Deleted Added
sdiff udiff text old ( 91420 ) new ( 92029 )
full compact
swap_pager.c (91420) swap_pager.c (92029)
1/*
2 * Copyright (c) 1998 Matthew Dillon,
3 * Copyright (c) 1994 John S. Dyson
4 * Copyright (c) 1990 University of Utah.
5 * Copyright (c) 1991, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by

--- 50 unchanged lines hidden (view full) ---

59 * cycled (in a high-load system) by the pager. We also do on-the-fly
60 * removal of invalidated swap blocks when a page is destroyed
61 * or renamed.
62 *
63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64 *
65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94
66 *
1/*
2 * Copyright (c) 1998 Matthew Dillon,
3 * Copyright (c) 1994 John S. Dyson
4 * Copyright (c) 1990 University of Utah.
5 * Copyright (c) 1991, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by

--- 50 unchanged lines hidden (view full) ---

59 * cycled (in a high-load system) by the pager. We also do on-the-fly
60 * removal of invalidated swap blocks when a page is destroyed
61 * or renamed.
62 *
63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64 *
65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94
66 *
67 * $FreeBSD: head/sys/vm/swap_pager.c 91420 2002-02-27 19:18:10Z jhb $
67 * $FreeBSD: head/sys/vm/swap_pager.c 92029 2002-03-10 21:52:48Z eivind $
68 */
69
70#include <sys/param.h>
71#include <sys/systm.h>
72#include <sys/conf.h>
73#include <sys/kernel.h>
74#include <sys/proc.h>
75#include <sys/bio.h>

--- 28 unchanged lines hidden (view full) ---

104
105#define SWM_FREE 0x02 /* free, period */
106#define SWM_POP 0x04 /* pop out */
107
108/*
109 * vm_swap_size is in page-sized chunks now. It was DEV_BSIZE'd chunks
110 * in the old system.
111 */
68 */
69
70#include <sys/param.h>
71#include <sys/systm.h>
72#include <sys/conf.h>
73#include <sys/kernel.h>
74#include <sys/proc.h>
75#include <sys/bio.h>

--- 28 unchanged lines hidden (view full) ---

104
105#define SWM_FREE 0x02 /* free, period */
106#define SWM_POP 0x04 /* pop out */
107
108/*
109 * vm_swap_size is in page-sized chunks now. It was DEV_BSIZE'd chunks
110 * in the old system.
111 */
112
113extern int vm_swap_size; /* number of free swap blocks, in pages */
114
115int swap_pager_full; /* swap space exhaustion (task killing) */
116static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/
117static int nsw_rcount; /* free read buffers */
118static int nsw_wcount_sync; /* limit write buffers / synchronous */
119static int nsw_wcount_async; /* limit write buffers / asynchronous */
120static int nsw_wcount_async_max;/* assigned maximum */

--- 30 unchanged lines hidden (view full) ---

151struct pagerlst swap_pager_un_object_list;
152vm_zone_t swap_zone;
153
154/*
155 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure
156 * calls hooked from other parts of the VM system and do not appear here.
157 * (see vm/swap_pager.h).
158 */
112extern int vm_swap_size; /* number of free swap blocks, in pages */
113
114int swap_pager_full; /* swap space exhaustion (task killing) */
115static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/
116static int nsw_rcount; /* free read buffers */
117static int nsw_wcount_sync; /* limit write buffers / synchronous */
118static int nsw_wcount_async; /* limit write buffers / asynchronous */
119static int nsw_wcount_async_max;/* assigned maximum */

--- 30 unchanged lines hidden (view full) ---

150struct pagerlst swap_pager_un_object_list;
151vm_zone_t swap_zone;
152
153/*
154 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure
155 * calls hooked from other parts of the VM system and do not appear here.
156 * (see vm/swap_pager.h).
157 */
159
160static vm_object_t
161 swap_pager_alloc __P((void *handle, vm_ooffset_t size,
162 vm_prot_t prot, vm_ooffset_t offset));
163static void swap_pager_dealloc __P((vm_object_t object));
164static int swap_pager_getpages __P((vm_object_t, vm_page_t *, int, int));
165static void swap_pager_init __P((void));
166static void swap_pager_unswapped __P((vm_page_t));
167static void swap_pager_strategy __P((vm_object_t, struct bio *));

--- 15 unchanged lines hidden (view full) ---

183
184/*
185 * dmmax is in page-sized chunks with the new swap system. It was
186 * dev-bsized chunks in the old. dmmax is always a power of 2.
187 *
188 * swap_*() routines are externally accessible. swp_*() routines are
189 * internal.
190 */
158static vm_object_t
159 swap_pager_alloc __P((void *handle, vm_ooffset_t size,
160 vm_prot_t prot, vm_ooffset_t offset));
161static void swap_pager_dealloc __P((vm_object_t object));
162static int swap_pager_getpages __P((vm_object_t, vm_page_t *, int, int));
163static void swap_pager_init __P((void));
164static void swap_pager_unswapped __P((vm_page_t));
165static void swap_pager_strategy __P((vm_object_t, struct bio *));

--- 15 unchanged lines hidden (view full) ---

181
182/*
183 * dmmax is in page-sized chunks with the new swap system. It was
184 * dev-bsized chunks in the old. dmmax is always a power of 2.
185 *
186 * swap_*() routines are externally accessible. swp_*() routines are
187 * internal.
188 */
191
192int dmmax;
193static int dmmax_mask;
194int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */
195int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */
196
197SYSCTL_INT(_vm, OID_AUTO, dmmax,
198 CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
199
200static __inline void swp_sizecheck __P((void));
201static void swp_pager_sync_iodone __P((struct buf *bp));
202static void swp_pager_async_iodone __P((struct buf *bp));
203
204/*
205 * Swap bitmap functions
206 */
189int dmmax;
190static int dmmax_mask;
191int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */
192int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */
193
194SYSCTL_INT(_vm, OID_AUTO, dmmax,
195 CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
196
197static __inline void swp_sizecheck __P((void));
198static void swp_pager_sync_iodone __P((struct buf *bp));
199static void swp_pager_async_iodone __P((struct buf *bp));
200
201/*
202 * Swap bitmap functions
203 */
207
208static __inline void swp_pager_freeswapspace __P((daddr_t blk, int npages));
209static __inline daddr_t swp_pager_getswapspace __P((int npages));
210
211/*
212 * Metadata functions
213 */
204static __inline void swp_pager_freeswapspace __P((daddr_t blk, int npages));
205static __inline daddr_t swp_pager_getswapspace __P((int npages));
206
207/*
208 * Metadata functions
209 */
214
215static void swp_pager_meta_build __P((vm_object_t, vm_pindex_t, daddr_t));
216static void swp_pager_meta_free __P((vm_object_t, vm_pindex_t, daddr_t));
217static void swp_pager_meta_free_all __P((vm_object_t));
218static daddr_t swp_pager_meta_ctl __P((vm_object_t, vm_pindex_t, int));
219
220/*
221 * SWP_SIZECHECK() - update swap_pager_full indication
222 *
223 * update the swap_pager_almost_full indication and warn when we are
224 * about to run out of swap space, using lowat/hiwat hysteresis.
225 *
226 * Clear swap_pager_full ( task killing ) indication when lowat is met.
227 *
228 * No restrictions on call
229 * This routine may not block.
230 * This routine must be called at splvm()
231 */
210static void swp_pager_meta_build __P((vm_object_t, vm_pindex_t, daddr_t));
211static void swp_pager_meta_free __P((vm_object_t, vm_pindex_t, daddr_t));
212static void swp_pager_meta_free_all __P((vm_object_t));
213static daddr_t swp_pager_meta_ctl __P((vm_object_t, vm_pindex_t, int));
214
215/*
216 * SWP_SIZECHECK() - update swap_pager_full indication
217 *
218 * update the swap_pager_almost_full indication and warn when we are
219 * about to run out of swap space, using lowat/hiwat hysteresis.
220 *
221 * Clear swap_pager_full ( task killing ) indication when lowat is met.
222 *
223 * No restrictions on call
224 * This routine may not block.
225 * This routine must be called at splvm()
226 */
232
233static __inline void
234swp_sizecheck()
235{
236 GIANT_REQUIRED;
237
238 if (vm_swap_size < nswap_lowat) {
239 if (swap_pager_almost_full == 0) {
240 printf("swap_pager: out of swap space\n");

--- 8 unchanged lines hidden (view full) ---

249
250/*
251 * SWAP_PAGER_INIT() - initialize the swap pager!
252 *
253 * Expected to be started from system init. NOTE: This code is run
254 * before much else so be careful what you depend on. Most of the VM
255 * system has yet to be initialized at this point.
256 */
227static __inline void
228swp_sizecheck()
229{
230 GIANT_REQUIRED;
231
232 if (vm_swap_size < nswap_lowat) {
233 if (swap_pager_almost_full == 0) {
234 printf("swap_pager: out of swap space\n");

--- 8 unchanged lines hidden (view full) ---

243
244/*
245 * SWAP_PAGER_INIT() - initialize the swap pager!
246 *
247 * Expected to be started from system init. NOTE: This code is run
248 * before much else so be careful what you depend on. Most of the VM
249 * system has yet to be initialized at this point.
250 */
257
258static void
259swap_pager_init()
260{
261 /*
262 * Initialize object lists
263 */
264 int i;
265
266 for (i = 0; i < NOBJLISTS; ++i)
267 TAILQ_INIT(&swap_pager_object_list[i]);
268 TAILQ_INIT(&swap_pager_un_object_list);
269 mtx_init(&sw_alloc_mtx, "swap_pager list", MTX_DEF);
270
271 /*
272 * Device Stripe, in PAGE_SIZE'd blocks
273 */
251static void
252swap_pager_init()
253{
254 /*
255 * Initialize object lists
256 */
257 int i;
258
259 for (i = 0; i < NOBJLISTS; ++i)
260 TAILQ_INIT(&swap_pager_object_list[i]);
261 TAILQ_INIT(&swap_pager_un_object_list);
262 mtx_init(&sw_alloc_mtx, "swap_pager list", MTX_DEF);
263
264 /*
265 * Device Stripe, in PAGE_SIZE'd blocks
266 */
274
275 dmmax = SWB_NPAGES * 2;
276 dmmax_mask = ~(dmmax - 1);
277}
278
279/*
280 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
281 *
282 * Expected to be started from pageout process once, prior to entering
283 * its main loop.
284 */
267 dmmax = SWB_NPAGES * 2;
268 dmmax_mask = ~(dmmax - 1);
269}
270
271/*
272 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
273 *
274 * Expected to be started from pageout process once, prior to entering
275 * its main loop.
276 */
285
286void
287swap_pager_swap_init()
288{
289 int n, n2;
290
291 /*
292 * Number of in-transit swap bp operations. Don't
293 * exhaust the pbufs completely. Make sure we

--- 11 unchanged lines hidden (view full) ---

305 * devices but is probably a little low if you have more. Even so,
306 * a higher value would probably generate only a limited improvement
307 * with three or four active swap devices since the system does not
308 * typically have to pageout at extreme bandwidths. We will want
309 * at least 2 per swap devices, and 4 is a pretty good value if you
310 * have one NFS swap device due to the command/ack latency over NFS.
311 * So it all works out pretty well.
312 */
277void
278swap_pager_swap_init()
279{
280 int n, n2;
281
282 /*
283 * Number of in-transit swap bp operations. Don't
284 * exhaust the pbufs completely. Make sure we

--- 11 unchanged lines hidden (view full) ---

296 * devices but is probably a little low if you have more. Even so,
297 * a higher value would probably generate only a limited improvement
298 * with three or four active swap devices since the system does not
299 * typically have to pageout at extreme bandwidths. We will want
300 * at least 2 per swap devices, and 4 is a pretty good value if you
301 * have one NFS swap device due to the command/ack latency over NFS.
302 * So it all works out pretty well.
303 */
313
314 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
315
316 mtx_lock(&pbuf_mtx);
317 nsw_rcount = (nswbuf + 1) / 2;
318 nsw_wcount_sync = (nswbuf + 3) / 4;
319 nsw_wcount_async = 4;
320 nsw_wcount_async_max = nsw_wcount_async;
321 mtx_unlock(&pbuf_mtx);
322
323 /*
324 * Initialize our zone. Right now I'm just guessing on the number
325 * we need based on the number of pages in the system. Each swblock
326 * can hold 16 pages, so this is probably overkill. This reservation
327 * is typically limited to around 70MB by default.
328 */
304 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
305
306 mtx_lock(&pbuf_mtx);
307 nsw_rcount = (nswbuf + 1) / 2;
308 nsw_wcount_sync = (nswbuf + 3) / 4;
309 nsw_wcount_async = 4;
310 nsw_wcount_async_max = nsw_wcount_async;
311 mtx_unlock(&pbuf_mtx);
312
313 /*
314 * Initialize our zone. Right now I'm just guessing on the number
315 * we need based on the number of pages in the system. Each swblock
316 * can hold 16 pages, so this is probably overkill. This reservation
317 * is typically limited to around 70MB by default.
318 */
329
330 n = cnt.v_page_count;
331 if (maxswzone && n > maxswzone / sizeof(struct swblock))
332 n = maxswzone / sizeof(struct swblock);
333 n2 = n;
319 n = cnt.v_page_count;
320 if (maxswzone && n > maxswzone / sizeof(struct swblock))
321 n = maxswzone / sizeof(struct swblock);
322 n2 = n;
334
335 do {
336 swap_zone = zinit(
337 "SWAPMETA",
338 sizeof(struct swblock),
339 n,
340 ZONE_INTERRUPT,
341 1
342 );
343 if (swap_zone != NULL)
344 break;
345 /*
346 * if the allocation failed, try a zone two thirds the
347 * size of the previous attempt.
348 */
349 n -= ((n + 2) / 3);
350 } while (n > 0);
323 do {
324 swap_zone = zinit(
325 "SWAPMETA",
326 sizeof(struct swblock),
327 n,
328 ZONE_INTERRUPT,
329 1
330 );
331 if (swap_zone != NULL)
332 break;
333 /*
334 * if the allocation failed, try a zone two thirds the
335 * size of the previous attempt.
336 */
337 n -= ((n + 2) / 3);
338 } while (n > 0);
351
352 if (swap_zone == NULL)
353 panic("failed to zinit swap_zone.");
354 if (n2 != n)
355 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
356 n2 = n;
357
358 /*
359 * Initialize our meta-data hash table. The swapper does not need to
360 * be quite as efficient as the VM system, so we do not use an
361 * oversized hash table.
362 *
363 * n: size of hash table, must be power of 2
364 * swhash_mask: hash table index mask
365 */
339 if (swap_zone == NULL)
340 panic("failed to zinit swap_zone.");
341 if (n2 != n)
342 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
343 n2 = n;
344
345 /*
346 * Initialize our meta-data hash table. The swapper does not need to
347 * be quite as efficient as the VM system, so we do not use an
348 * oversized hash table.
349 *
350 * n: size of hash table, must be power of 2
351 * swhash_mask: hash table index mask
352 */
366
367 for (n = 1; n < n2 / 8; n *= 2)
368 ;
353 for (n = 1; n < n2 / 8; n *= 2)
354 ;
369
370 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
355 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
371
372 swhash_mask = n - 1;
373}
374
375/*
376 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
377 * its metadata structures.
378 *
379 * This routine is called from the mmap and fork code to create a new
380 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
381 * and then converting it with swp_pager_meta_build().
382 *
383 * This routine may block in vm_object_allocate() and create a named
384 * object lookup race, so we must interlock. We must also run at
385 * splvm() for the object lookup to handle races with interrupts, but
386 * we do not have to maintain splvm() in between the lookup and the
387 * add because (I believe) it is not possible to attempt to create
388 * a new swap object w/handle when a default object with that handle
389 * already exists.
390 */
356 swhash_mask = n - 1;
357}
358
359/*
360 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
361 * its metadata structures.
362 *
363 * This routine is called from the mmap and fork code to create a new
364 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
365 * and then converting it with swp_pager_meta_build().
366 *
367 * This routine may block in vm_object_allocate() and create a named
368 * object lookup race, so we must interlock. We must also run at
369 * splvm() for the object lookup to handle races with interrupts, but
370 * we do not have to maintain splvm() in between the lookup and the
371 * add because (I believe) it is not possible to attempt to create
372 * a new swap object w/handle when a default object with that handle
373 * already exists.
374 */
391
392static vm_object_t
393swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
394 vm_ooffset_t offset)
395{
396 vm_object_t object;
397
398 GIANT_REQUIRED;
399

--- 34 unchanged lines hidden (view full) ---

434 * designed such that we can reinstantiate it later, but this
435 * routine is typically called only when the entire object is
436 * about to be destroyed.
437 *
438 * This routine may block, but no longer does.
439 *
440 * The object must be locked or unreferenceable.
441 */
375static vm_object_t
376swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
377 vm_ooffset_t offset)
378{
379 vm_object_t object;
380
381 GIANT_REQUIRED;
382

--- 34 unchanged lines hidden (view full) ---

417 * designed such that we can reinstantiate it later, but this
418 * routine is typically called only when the entire object is
419 * about to be destroyed.
420 *
421 * This routine may block, but no longer does.
422 *
423 * The object must be locked or unreferenceable.
424 */
442
443static void
444swap_pager_dealloc(object)
445 vm_object_t object;
446{
447 int s;
448
449 GIANT_REQUIRED;
450

--- 37 unchanged lines hidden (view full) ---

488 * when they configured swap and didn't configure enough.
489 *
490 * Must be called at splvm() to avoid races with bitmap frees from
491 * vm_page_remove() aka swap_pager_page_removed().
492 *
493 * This routine may not block
494 * This routine must be called at splvm().
495 */
425static void
426swap_pager_dealloc(object)
427 vm_object_t object;
428{
429 int s;
430
431 GIANT_REQUIRED;
432

--- 37 unchanged lines hidden (view full) ---

470 * when they configured swap and didn't configure enough.
471 *
472 * Must be called at splvm() to avoid races with bitmap frees from
473 * vm_page_remove() aka swap_pager_page_removed().
474 *
475 * This routine may not block
476 * This routine must be called at splvm().
477 */
496
497static __inline daddr_t
498swp_pager_getswapspace(npages)
499 int npages;
500{
501 daddr_t blk;
502
503 GIANT_REQUIRED;
504

--- 4 unchanged lines hidden (view full) ---

509 swap_pager_almost_full = 1;
510 }
511 } else {
512 vm_swap_size -= npages;
513 /* per-swap area stats */
514 swdevt[BLK2DEVIDX(blk)].sw_used += npages;
515 swp_sizecheck();
516 }
478static __inline daddr_t
479swp_pager_getswapspace(npages)
480 int npages;
481{
482 daddr_t blk;
483
484 GIANT_REQUIRED;
485

--- 4 unchanged lines hidden (view full) ---

490 swap_pager_almost_full = 1;
491 }
492 } else {
493 vm_swap_size -= npages;
494 /* per-swap area stats */
495 swdevt[BLK2DEVIDX(blk)].sw_used += npages;
496 swp_sizecheck();
497 }
517 return(blk);
498 return (blk);
518}
519
520/*
521 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
522 *
523 * This routine returns the specified swap blocks back to the bitmap.
524 *
525 * Note: This routine may not block (it could in the old swap code),
526 * and through the use of the new blist routines it does not block.
527 *
528 * We must be called at splvm() to avoid races with bitmap frees from
529 * vm_page_remove() aka swap_pager_page_removed().
530 *
531 * This routine may not block
532 * This routine must be called at splvm().
533 */
499}
500
501/*
502 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
503 *
504 * This routine returns the specified swap blocks back to the bitmap.
505 *
506 * Note: This routine may not block (it could in the old swap code),
507 * and through the use of the new blist routines it does not block.
508 *
509 * We must be called at splvm() to avoid races with bitmap frees from
510 * vm_page_remove() aka swap_pager_page_removed().
511 *
512 * This routine may not block
513 * This routine must be called at splvm().
514 */
534
535static __inline void
536swp_pager_freeswapspace(blk, npages)
537 daddr_t blk;
538 int npages;
539{
540 GIANT_REQUIRED;
541
542 blist_free(swapblist, blk, npages);

--- 13 unchanged lines hidden (view full) ---

556 *
557 * The external callers of this routine typically have already destroyed
558 * or renamed vm_page_t's associated with this range in the object so
559 * we should be ok.
560 *
561 * This routine may be called at any spl. We up our spl to splvm temporarily
562 * in order to perform the metadata removal.
563 */
515static __inline void
516swp_pager_freeswapspace(blk, npages)
517 daddr_t blk;
518 int npages;
519{
520 GIANT_REQUIRED;
521
522 blist_free(swapblist, blk, npages);

--- 13 unchanged lines hidden (view full) ---

536 *
537 * The external callers of this routine typically have already destroyed
538 * or renamed vm_page_t's associated with this range in the object so
539 * we should be ok.
540 *
541 * This routine may be called at any spl. We up our spl to splvm temporarily
542 * in order to perform the metadata removal.
543 */
564
565void
566swap_pager_freespace(object, start, size)
567 vm_object_t object;
568 vm_pindex_t start;
569 vm_size_t size;
570{
571 int s = splvm();
572

--- 5 unchanged lines hidden (view full) ---

578/*
579 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
580 *
581 * Assigns swap blocks to the specified range within the object. The
582 * swap blocks are not zerod. Any previous swap assignment is destroyed.
583 *
584 * Returns 0 on success, -1 on failure.
585 */
544void
545swap_pager_freespace(object, start, size)
546 vm_object_t object;
547 vm_pindex_t start;
548 vm_size_t size;
549{
550 int s = splvm();
551

--- 5 unchanged lines hidden (view full) ---

557/*
558 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
559 *
560 * Assigns swap blocks to the specified range within the object. The
561 * swap blocks are not zerod. Any previous swap assignment is destroyed.
562 *
563 * Returns 0 on success, -1 on failure.
564 */
586
587int
588swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
589{
590 int s;
591 int n = 0;
592 daddr_t blk = SWAPBLK_NONE;
593 vm_pindex_t beg = start; /* save start index */
594
595 s = splvm();
596 while (size) {
597 if (n == 0) {
598 n = BLIST_MAX_ALLOC;
599 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
600 n >>= 1;
601 if (n == 0) {
602 swp_pager_meta_free(object, beg, start - beg);
603 splx(s);
565int
566swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
567{
568 int s;
569 int n = 0;
570 daddr_t blk = SWAPBLK_NONE;
571 vm_pindex_t beg = start; /* save start index */
572
573 s = splvm();
574 while (size) {
575 if (n == 0) {
576 n = BLIST_MAX_ALLOC;
577 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
578 n >>= 1;
579 if (n == 0) {
580 swp_pager_meta_free(object, beg, start - beg);
581 splx(s);
604 return(-1);
582 return (-1);
605 }
606 }
607 }
608 swp_pager_meta_build(object, start, blk);
609 --size;
610 ++start;
611 ++blk;
612 --n;
613 }
614 swp_pager_meta_free(object, start, n);
615 splx(s);
583 }
584 }
585 }
586 swp_pager_meta_build(object, start, blk);
587 --size;
588 ++start;
589 ++blk;
590 --n;
591 }
592 swp_pager_meta_free(object, start, n);
593 splx(s);
616 return(0);
594 return (0);
617}
618
619/*
620 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
621 * and destroy the source.
622 *
623 * Copy any valid swapblks from the source to the destination. In
624 * cases where both the source and destination have a valid swapblk,

--- 12 unchanged lines hidden (view full) ---

637 *
638 * The source object contains no vm_page_t's (which is just as well)
639 *
640 * The source object is of type OBJT_SWAP.
641 *
642 * The source and destination objects must be locked or
643 * inaccessible (XXX are they ?)
644 */
595}
596
597/*
598 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
599 * and destroy the source.
600 *
601 * Copy any valid swapblks from the source to the destination. In
602 * cases where both the source and destination have a valid swapblk,

--- 12 unchanged lines hidden (view full) ---

615 *
616 * The source object contains no vm_page_t's (which is just as well)
617 *
618 * The source object is of type OBJT_SWAP.
619 *
620 * The source and destination objects must be locked or
621 * inaccessible (XXX are they ?)
622 */
645
646void
647swap_pager_copy(srcobject, dstobject, offset, destroysource)
648 vm_object_t srcobject;
649 vm_object_t dstobject;
650 vm_pindex_t offset;
651 int destroysource;
652{
653 vm_pindex_t i;
654 int s;
655
656 GIANT_REQUIRED;
657
658 s = splvm();
659 /*
660 * If destroysource is set, we remove the source object from the
661 * swap_pager internal queue now.
662 */
623void
624swap_pager_copy(srcobject, dstobject, offset, destroysource)
625 vm_object_t srcobject;
626 vm_object_t dstobject;
627 vm_pindex_t offset;
628 int destroysource;
629{
630 vm_pindex_t i;
631 int s;
632
633 GIANT_REQUIRED;
634
635 s = splvm();
636 /*
637 * If destroysource is set, we remove the source object from the
638 * swap_pager internal queue now.
639 */
663
664 if (destroysource) {
665 mtx_lock(&sw_alloc_mtx);
666 if (srcobject->handle == NULL) {
667 TAILQ_REMOVE(
668 &swap_pager_un_object_list,
669 srcobject,
670 pager_object_list
671 );

--- 5 unchanged lines hidden (view full) ---

677 );
678 }
679 mtx_unlock(&sw_alloc_mtx);
680 }
681
682 /*
683 * transfer source to destination.
684 */
640 if (destroysource) {
641 mtx_lock(&sw_alloc_mtx);
642 if (srcobject->handle == NULL) {
643 TAILQ_REMOVE(
644 &swap_pager_un_object_list,
645 srcobject,
646 pager_object_list
647 );

--- 5 unchanged lines hidden (view full) ---

653 );
654 }
655 mtx_unlock(&sw_alloc_mtx);
656 }
657
658 /*
659 * transfer source to destination.
660 */
685
686 for (i = 0; i < dstobject->size; ++i) {
687 daddr_t dstaddr;
688
689 /*
690 * Locate (without changing) the swapblk on the destination,
691 * unless it is invalid in which case free it silently, or
692 * if the destination is a resident page, in which case the
693 * source is thrown away.
694 */
661 for (i = 0; i < dstobject->size; ++i) {
662 daddr_t dstaddr;
663
664 /*
665 * Locate (without changing) the swapblk on the destination,
666 * unless it is invalid in which case free it silently, or
667 * if the destination is a resident page, in which case the
668 * source is thrown away.
669 */
695
696 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
697
698 if (dstaddr == SWAPBLK_NONE) {
699 /*
700 * Destination has no swapblk and is not resident,
701 * copy source.
702 */
703 daddr_t srcaddr;

--- 17 unchanged lines hidden (view full) ---

721 }
722
723 /*
724 * Free left over swap blocks in source.
725 *
726 * We have to revert the type to OBJT_DEFAULT so we do not accidently
727 * double-remove the object from the swap queues.
728 */
670 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
671
672 if (dstaddr == SWAPBLK_NONE) {
673 /*
674 * Destination has no swapblk and is not resident,
675 * copy source.
676 */
677 daddr_t srcaddr;

--- 17 unchanged lines hidden (view full) ---

695 }
696
697 /*
698 * Free left over swap blocks in source.
699 *
700 * We have to revert the type to OBJT_DEFAULT so we do not accidently
701 * double-remove the object from the swap queues.
702 */
729
730 if (destroysource) {
731 swp_pager_meta_free_all(srcobject);
732 /*
733 * Reverting the type is not necessary, the caller is going
734 * to destroy srcobject directly, but I'm doing it here
735 * for consistency since we've removed the object from its
736 * queues.
737 */

--- 10 unchanged lines hidden (view full) ---

748 * page and return TRUE if it does, FALSE if it doesn't.
749 *
750 * If TRUE, we also try to determine how much valid, contiguous backing
751 * store exists before and after the requested page within a reasonable
752 * distance. We do not try to restrict it to the swap device stripe
753 * (that is handled in getpages/putpages). It probably isn't worth
754 * doing here.
755 */
703 if (destroysource) {
704 swp_pager_meta_free_all(srcobject);
705 /*
706 * Reverting the type is not necessary, the caller is going
707 * to destroy srcobject directly, but I'm doing it here
708 * for consistency since we've removed the object from its
709 * queues.
710 */

--- 10 unchanged lines hidden (view full) ---

721 * page and return TRUE if it does, FALSE if it doesn't.
722 *
723 * If TRUE, we also try to determine how much valid, contiguous backing
724 * store exists before and after the requested page within a reasonable
725 * distance. We do not try to restrict it to the swap device stripe
726 * (that is handled in getpages/putpages). It probably isn't worth
727 * doing here.
728 */
756
757boolean_t
758swap_pager_haspage(object, pindex, before, after)
759 vm_object_t object;
760 vm_pindex_t pindex;
761 int *before;
762 int *after;
763{
764 daddr_t blk0;
765 int s;
766
767 /*
768 * do we have good backing store at the requested index ?
769 */
729boolean_t
730swap_pager_haspage(object, pindex, before, after)
731 vm_object_t object;
732 vm_pindex_t pindex;
733 int *before;
734 int *after;
735{
736 daddr_t blk0;
737 int s;
738
739 /*
740 * do we have good backing store at the requested index ?
741 */
770
771 s = splvm();
772 blk0 = swp_pager_meta_ctl(object, pindex, 0);
773
774 if (blk0 == SWAPBLK_NONE) {
775 splx(s);
776 if (before)
777 *before = 0;
778 if (after)
779 *after = 0;
780 return (FALSE);
781 }
782
783 /*
784 * find backwards-looking contiguous good backing store
785 */
742 s = splvm();
743 blk0 = swp_pager_meta_ctl(object, pindex, 0);
744
745 if (blk0 == SWAPBLK_NONE) {
746 splx(s);
747 if (before)
748 *before = 0;
749 if (after)
750 *after = 0;
751 return (FALSE);
752 }
753
754 /*
755 * find backwards-looking contiguous good backing store
756 */
786
787 if (before != NULL) {
788 int i;
789
790 for (i = 1; i < (SWB_NPAGES/2); ++i) {
791 daddr_t blk;
792
793 if (i > pindex)
794 break;
795 blk = swp_pager_meta_ctl(object, pindex - i, 0);
796 if (blk != blk0 - i)
797 break;
798 }
799 *before = (i - 1);
800 }
801
802 /*
803 * find forward-looking contiguous good backing store
804 */
757 if (before != NULL) {
758 int i;
759
760 for (i = 1; i < (SWB_NPAGES/2); ++i) {
761 daddr_t blk;
762
763 if (i > pindex)
764 break;
765 blk = swp_pager_meta_ctl(object, pindex - i, 0);
766 if (blk != blk0 - i)
767 break;
768 }
769 *before = (i - 1);
770 }
771
772 /*
773 * find forward-looking contiguous good backing store
774 */
805
806 if (after != NULL) {
807 int i;
808
809 for (i = 1; i < (SWB_NPAGES/2); ++i) {
810 daddr_t blk;
811
812 blk = swp_pager_meta_ctl(object, pindex + i, 0);
813 if (blk != blk0 + i)

--- 18 unchanged lines hidden (view full) ---

832 * NOTE!!! If the page is clean and the swap was valid, the caller
833 * should make the page dirty before calling this routine. This routine
834 * does NOT change the m->dirty status of the page. Also: MADV_FREE
835 * depends on it.
836 *
837 * This routine may not block
838 * This routine must be called at splvm()
839 */
775 if (after != NULL) {
776 int i;
777
778 for (i = 1; i < (SWB_NPAGES/2); ++i) {
779 daddr_t blk;
780
781 blk = swp_pager_meta_ctl(object, pindex + i, 0);
782 if (blk != blk0 + i)

--- 18 unchanged lines hidden (view full) ---

801 * NOTE!!! If the page is clean and the swap was valid, the caller
802 * should make the page dirty before calling this routine. This routine
803 * does NOT change the m->dirty status of the page. Also: MADV_FREE
804 * depends on it.
805 *
806 * This routine may not block
807 * This routine must be called at splvm()
808 */
840
841static void
842swap_pager_unswapped(m)
843 vm_page_t m;
844{
845 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
846}
847
848/*

--- 8 unchanged lines hidden (view full) ---

857 *
858 * Note that b_blkno is scaled for PAGE_SIZE
859 *
860 * We currently attempt to run I/O synchronously or asynchronously as
861 * the caller requests. This isn't perfect because we loose error
862 * sequencing when we run multiple ops in parallel to satisfy a request.
863 * But this is swap, so we let it all hang out.
864 */
809static void
810swap_pager_unswapped(m)
811 vm_page_t m;
812{
813 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
814}
815
816/*

--- 8 unchanged lines hidden (view full) ---

825 *
826 * Note that b_blkno is scaled for PAGE_SIZE
827 *
828 * We currently attempt to run I/O synchronously or asynchronously as
829 * the caller requests. This isn't perfect because we loose error
830 * sequencing when we run multiple ops in parallel to satisfy a request.
831 * But this is swap, so we let it all hang out.
832 */
865
866static void
867swap_pager_strategy(vm_object_t object, struct bio *bp)
868{
869 vm_pindex_t start;
870 int count;
871 int s;
872 char *data;
873 struct buf *nbp = NULL;

--- 5 unchanged lines hidden (view full) ---

879 biofinish(bp, NULL, EINVAL);
880 printf("swap_pager_strategy: bp %p blk %d size %d, not page bounded\n", bp, (int)bp->bio_pblkno, (int)bp->bio_bcount);
881 return;
882 }
883
884 /*
885 * Clear error indication, initialize page index, count, data pointer.
886 */
833static void
834swap_pager_strategy(vm_object_t object, struct bio *bp)
835{
836 vm_pindex_t start;
837 int count;
838 int s;
839 char *data;
840 struct buf *nbp = NULL;

--- 5 unchanged lines hidden (view full) ---

846 biofinish(bp, NULL, EINVAL);
847 printf("swap_pager_strategy: bp %p blk %d size %d, not page bounded\n", bp, (int)bp->bio_pblkno, (int)bp->bio_bcount);
848 return;
849 }
850
851 /*
852 * Clear error indication, initialize page index, count, data pointer.
853 */
887
888 bp->bio_error = 0;
889 bp->bio_flags &= ~BIO_ERROR;
890 bp->bio_resid = bp->bio_bcount;
891 *(u_int *) &bp->bio_driver1 = 0;
892
893 start = bp->bio_pblkno;
894 count = howmany(bp->bio_bcount, PAGE_SIZE);
895 data = bp->bio_data;
896
897 s = splvm();
898
899 /*
900 * Deal with BIO_DELETE
901 */
854 bp->bio_error = 0;
855 bp->bio_flags &= ~BIO_ERROR;
856 bp->bio_resid = bp->bio_bcount;
857 *(u_int *) &bp->bio_driver1 = 0;
858
859 start = bp->bio_pblkno;
860 count = howmany(bp->bio_bcount, PAGE_SIZE);
861 data = bp->bio_data;
862
863 s = splvm();
864
865 /*
866 * Deal with BIO_DELETE
867 */
902
903 if (bp->bio_cmd == BIO_DELETE) {
904 /*
905 * FREE PAGE(s) - destroy underlying swap that is no longer
906 * needed.
907 */
908 swp_pager_meta_free(object, start, count);
909 splx(s);
910 bp->bio_resid = 0;

--- 26 unchanged lines hidden (view full) ---

937 /*
938 * Do we have to flush our current collection? Yes if:
939 *
940 * - no swap block at this index
941 * - swap block is not contiguous
942 * - we cross a physical disk boundry in the
943 * stripe.
944 */
868 if (bp->bio_cmd == BIO_DELETE) {
869 /*
870 * FREE PAGE(s) - destroy underlying swap that is no longer
871 * needed.
872 */
873 swp_pager_meta_free(object, start, count);
874 splx(s);
875 bp->bio_resid = 0;

--- 26 unchanged lines hidden (view full) ---

902 /*
903 * Do we have to flush our current collection? Yes if:
904 *
905 * - no swap block at this index
906 * - swap block is not contiguous
907 * - we cross a physical disk boundry in the
908 * stripe.
909 */
945
946 if (
947 nbp && (nbp->b_blkno + btoc(nbp->b_bcount) != blk ||
948 ((nbp->b_blkno ^ blk) & dmmax_mask)
949 )
950 ) {
951 splx(s);
952 if (bp->bio_cmd == BIO_READ) {
953 ++cnt.v_swapin;

--- 7 unchanged lines hidden (view full) ---

961 s = splvm();
962 nbp = NULL;
963 }
964
965 /*
966 * Add new swapblk to nbp, instantiating nbp if necessary.
967 * Zero-fill reads are able to take a shortcut.
968 */
910 if (
911 nbp && (nbp->b_blkno + btoc(nbp->b_bcount) != blk ||
912 ((nbp->b_blkno ^ blk) & dmmax_mask)
913 )
914 ) {
915 splx(s);
916 if (bp->bio_cmd == BIO_READ) {
917 ++cnt.v_swapin;

--- 7 unchanged lines hidden (view full) ---

925 s = splvm();
926 nbp = NULL;
927 }
928
929 /*
930 * Add new swapblk to nbp, instantiating nbp if necessary.
931 * Zero-fill reads are able to take a shortcut.
932 */
969
970 if (blk == SWAPBLK_NONE) {
971 /*
972 * We can only get here if we are reading. Since
973 * we are at splvm() we can safely modify b_resid,
974 * even if chain ops are in progress.
975 */
976 bzero(data, PAGE_SIZE);
977 bp->bio_resid -= PAGE_SIZE;

--- 9 unchanged lines hidden (view full) ---

987 --count;
988 ++start;
989 data += PAGE_SIZE;
990 }
991
992 /*
993 * Flush out last buffer
994 */
933 if (blk == SWAPBLK_NONE) {
934 /*
935 * We can only get here if we are reading. Since
936 * we are at splvm() we can safely modify b_resid,
937 * even if chain ops are in progress.
938 */
939 bzero(data, PAGE_SIZE);
940 bp->bio_resid -= PAGE_SIZE;

--- 9 unchanged lines hidden (view full) ---

950 --count;
951 ++start;
952 data += PAGE_SIZE;
953 }
954
955 /*
956 * Flush out last buffer
957 */
995
996 splx(s);
997
998 if (nbp) {
999 if (nbp->b_iocmd == BIO_READ) {
1000 ++cnt.v_swapin;
1001 cnt.v_swappgsin += btoc(nbp->b_bcount);
1002 } else {
1003 ++cnt.v_swapout;
1004 cnt.v_swappgsout += btoc(nbp->b_bcount);
1005 nbp->b_dirtyend = nbp->b_bcount;
1006 }
1007 flushchainbuf(nbp);
1008 /* nbp = NULL; */
1009 }
1010 /*
1011 * Wait for completion.
1012 */
958 splx(s);
959
960 if (nbp) {
961 if (nbp->b_iocmd == BIO_READ) {
962 ++cnt.v_swapin;
963 cnt.v_swappgsin += btoc(nbp->b_bcount);
964 } else {
965 ++cnt.v_swapout;
966 cnt.v_swappgsout += btoc(nbp->b_bcount);
967 nbp->b_dirtyend = nbp->b_bcount;
968 }
969 flushchainbuf(nbp);
970 /* nbp = NULL; */
971 }
972 /*
973 * Wait for completion.
974 */
1013
1014 waitchainbuf(bp, 0, 1);
1015}
1016
1017/*
1018 * SWAP_PAGER_GETPAGES() - bring pages in from swap
1019 *
1020 * Attempt to retrieve (m, count) pages from backing store, but make
1021 * sure we retrieve at least m[reqpage]. We try to load in as large

--- 6 unchanged lines hidden (view full) ---

1028 * feature, I intend to improve on it in the future.
1029 *
1030 * The parent has a single vm_object_pip_add() reference prior to
1031 * calling us and we should return with the same.
1032 *
1033 * The parent has BUSY'd the pages. We should return with 'm'
1034 * left busy, but the others adjusted.
1035 */
975 waitchainbuf(bp, 0, 1);
976}
977
978/*
979 * SWAP_PAGER_GETPAGES() - bring pages in from swap
980 *
981 * Attempt to retrieve (m, count) pages from backing store, but make
982 * sure we retrieve at least m[reqpage]. We try to load in as large

--- 6 unchanged lines hidden (view full) ---

989 * feature, I intend to improve on it in the future.
990 *
991 * The parent has a single vm_object_pip_add() reference prior to
992 * calling us and we should return with the same.
993 *
994 * The parent has BUSY'd the pages. We should return with 'm'
995 * left busy, but the others adjusted.
996 */
1036
1037static int
1038swap_pager_getpages(object, m, count, reqpage)
1039 vm_object_t object;
1040 vm_page_t *m;
1041 int count, reqpage;
1042{
1043 struct buf *bp;
1044 vm_page_t mreq;

--- 19 unchanged lines hidden (view full) ---

1064 * their swapblks. We require a *contiguous* range that falls entirely
1065 * within a single device stripe. If we do not supply it, bad things
1066 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1067 * loops are set up such that the case(s) are handled implicitly.
1068 *
1069 * The swp_*() calls must be made at splvm(). vm_page_free() does
1070 * not need to be, but it will go a little faster if it is.
1071 */
997static int
998swap_pager_getpages(object, m, count, reqpage)
999 vm_object_t object;
1000 vm_page_t *m;
1001 int count, reqpage;
1002{
1003 struct buf *bp;
1004 vm_page_t mreq;

--- 19 unchanged lines hidden (view full) ---

1024 * their swapblks. We require a *contiguous* range that falls entirely
1025 * within a single device stripe. If we do not supply it, bad things
1026 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1027 * loops are set up such that the case(s) are handled implicitly.
1028 *
1029 * The swp_*() calls must be made at splvm(). vm_page_free() does
1030 * not need to be, but it will go a little faster if it is.
1031 */
1072
1073 s = splvm();
1074 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1075
1076 for (i = reqpage - 1; i >= 0; --i) {
1077 daddr_t iblk;
1078
1079 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1080 if (blk != iblk + (reqpage - i))

--- 12 unchanged lines hidden (view full) ---

1093 if ((blk ^ jblk) & dmmax_mask)
1094 break;
1095 }
1096
1097 /*
1098 * free pages outside our collection range. Note: we never free
1099 * mreq, it must remain busy throughout.
1100 */
1032 s = splvm();
1033 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1034
1035 for (i = reqpage - 1; i >= 0; --i) {
1036 daddr_t iblk;
1037
1038 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1039 if (blk != iblk + (reqpage - i))

--- 12 unchanged lines hidden (view full) ---

1052 if ((blk ^ jblk) & dmmax_mask)
1053 break;
1054 }
1055
1056 /*
1057 * free pages outside our collection range. Note: we never free
1058 * mreq, it must remain busy throughout.
1059 */
1101
1102 {
1103 int k;
1104
1105 for (k = 0; k < i; ++k)
1106 vm_page_free(m[k]);
1107 for (k = j; k < count; ++k)
1108 vm_page_free(m[k]);
1109 }
1110 splx(s);
1111
1112
1113 /*
1114 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1115 * still busy, but the others unbusied.
1116 */
1060 {
1061 int k;
1062
1063 for (k = 0; k < i; ++k)
1064 vm_page_free(m[k]);
1065 for (k = j; k < count; ++k)
1066 vm_page_free(m[k]);
1067 }
1068 splx(s);
1069
1070
1071 /*
1072 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1073 * still busy, but the others unbusied.
1074 */
1117
1118 if (blk == SWAPBLK_NONE)
1075 if (blk == SWAPBLK_NONE)
1119 return(VM_PAGER_FAIL);
1076 return (VM_PAGER_FAIL);
1120
1121 /*
1122 * Get a swap buffer header to perform the IO
1123 */
1077
1078 /*
1079 * Get a swap buffer header to perform the IO
1080 */
1124
1125 bp = getpbuf(&nsw_rcount);
1126 kva = (vm_offset_t) bp->b_data;
1127
1128 /*
1129 * map our page(s) into kva for input
1130 *
1131 * NOTE: B_PAGING is set by pbgetvp()
1132 */
1081 bp = getpbuf(&nsw_rcount);
1082 kva = (vm_offset_t) bp->b_data;
1083
1084 /*
1085 * map our page(s) into kva for input
1086 *
1087 * NOTE: B_PAGING is set by pbgetvp()
1088 */
1133
1134 pmap_qenter(kva, m + i, j - i);
1135
1136 bp->b_iocmd = BIO_READ;
1137 bp->b_iodone = swp_pager_async_iodone;
1138 bp->b_rcred = crhold(thread0.td_ucred);
1139 bp->b_wcred = crhold(thread0.td_ucred);
1140 bp->b_data = (caddr_t) kva;
1141 bp->b_blkno = blk - (reqpage - i);

--- 15 unchanged lines hidden (view full) ---

1157
1158 cnt.v_swapin++;
1159 cnt.v_swappgsin += bp->b_npages;
1160
1161 /*
1162 * We still hold the lock on mreq, and our automatic completion routine
1163 * does not remove it.
1164 */
1089 pmap_qenter(kva, m + i, j - i);
1090
1091 bp->b_iocmd = BIO_READ;
1092 bp->b_iodone = swp_pager_async_iodone;
1093 bp->b_rcred = crhold(thread0.td_ucred);
1094 bp->b_wcred = crhold(thread0.td_ucred);
1095 bp->b_data = (caddr_t) kva;
1096 bp->b_blkno = blk - (reqpage - i);

--- 15 unchanged lines hidden (view full) ---

1112
1113 cnt.v_swapin++;
1114 cnt.v_swappgsin += bp->b_npages;
1115
1116 /*
1117 * We still hold the lock on mreq, and our automatic completion routine
1118 * does not remove it.
1119 */
1165
1166 vm_object_pip_add(mreq->object, bp->b_npages);
1167 lastpindex = m[j-1]->pindex;
1168
1169 /*
1170 * perform the I/O. NOTE!!! bp cannot be considered valid after
1171 * this point because we automatically release it on completion.
1172 * Instead, we look at the one page we are interested in which we
1173 * still hold a lock on even through the I/O completion.

--- 6 unchanged lines hidden (view full) ---

1180 BUF_KERNPROC(bp);
1181 BUF_STRATEGY(bp);
1182
1183 /*
1184 * wait for the page we want to complete. PG_SWAPINPROG is always
1185 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1186 * is set in the meta-data.
1187 */
1120 vm_object_pip_add(mreq->object, bp->b_npages);
1121 lastpindex = m[j-1]->pindex;
1122
1123 /*
1124 * perform the I/O. NOTE!!! bp cannot be considered valid after
1125 * this point because we automatically release it on completion.
1126 * Instead, we look at the one page we are interested in which we
1127 * still hold a lock on even through the I/O completion.

--- 6 unchanged lines hidden (view full) ---

1134 BUF_KERNPROC(bp);
1135 BUF_STRATEGY(bp);
1136
1137 /*
1138 * wait for the page we want to complete. PG_SWAPINPROG is always
1139 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1140 * is set in the meta-data.
1141 */
1188
1189 s = splvm();
1142 s = splvm();
1190
1191 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1192 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1193 cnt.v_intrans++;
1194 if (tsleep(mreq, PSWP, "swread", hz*20)) {
1195 printf(
1196 "swap_pager: indefinite wait buffer: device:"
1197 " %s, blkno: %ld, size: %ld\n",
1198 devtoname(bp->b_dev), (long)bp->b_blkno,
1199 bp->b_bcount
1200 );
1201 }
1202 }
1143 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1144 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1145 cnt.v_intrans++;
1146 if (tsleep(mreq, PSWP, "swread", hz*20)) {
1147 printf(
1148 "swap_pager: indefinite wait buffer: device:"
1149 " %s, blkno: %ld, size: %ld\n",
1150 devtoname(bp->b_dev), (long)bp->b_blkno,
1151 bp->b_bcount
1152 );
1153 }
1154 }
1203
1204 splx(s);
1205
1206 /*
1155 splx(s);
1156
1157 /*
1207 * mreq is left bussied after completion, but all the other pages
1158 * mreq is left busied after completion, but all the other pages
1208 * are freed. If we had an unrecoverable read error the page will
1209 * not be valid.
1210 */
1159 * are freed. If we had an unrecoverable read error the page will
1160 * not be valid.
1161 */
1211
1212 if (mreq->valid != VM_PAGE_BITS_ALL) {
1162 if (mreq->valid != VM_PAGE_BITS_ALL) {
1213 return(VM_PAGER_ERROR);
1163 return (VM_PAGER_ERROR);
1214 } else {
1164 } else {
1215 return(VM_PAGER_OK);
1165 return (VM_PAGER_OK);
1216 }
1217
1218 /*
1219 * A final note: in a low swap situation, we cannot deallocate swap
1220 * and mark a page dirty here because the caller is likely to mark
1221 * the page clean when we return, causing the page to possibly revert
1222 * to all-zero's later.
1223 */

--- 16 unchanged lines hidden (view full) ---

1240 * calling us and will remove references for rtvals[] that are
1241 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1242 * completion.
1243 *
1244 * The parent has soft-busy'd the pages it passes us and will unbusy
1245 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1246 * We need to unbusy the rest on I/O completion.
1247 */
1166 }
1167
1168 /*
1169 * A final note: in a low swap situation, we cannot deallocate swap
1170 * and mark a page dirty here because the caller is likely to mark
1171 * the page clean when we return, causing the page to possibly revert
1172 * to all-zero's later.
1173 */

--- 16 unchanged lines hidden (view full) ---

1190 * calling us and will remove references for rtvals[] that are
1191 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1192 * completion.
1193 *
1194 * The parent has soft-busy'd the pages it passes us and will unbusy
1195 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1196 * We need to unbusy the rest on I/O completion.
1197 */
1248
1249void
1250swap_pager_putpages(object, m, count, sync, rtvals)
1251 vm_object_t object;
1252 vm_page_t *m;
1253 int count;
1254 boolean_t sync;
1255 int *rtvals;
1256{

--- 9 unchanged lines hidden (view full) ---

1266 }
1267 /*
1268 * Step 1
1269 *
1270 * Turn object into OBJT_SWAP
1271 * check for bogus sysops
1272 * force sync if not pageout process
1273 */
1198void
1199swap_pager_putpages(object, m, count, sync, rtvals)
1200 vm_object_t object;
1201 vm_page_t *m;
1202 int count;
1203 boolean_t sync;
1204 int *rtvals;
1205{

--- 9 unchanged lines hidden (view full) ---

1215 }
1216 /*
1217 * Step 1
1218 *
1219 * Turn object into OBJT_SWAP
1220 * check for bogus sysops
1221 * force sync if not pageout process
1222 */
1274
1275 if (object->type != OBJT_SWAP)
1276 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1277
1278 if (curproc != pageproc)
1279 sync = TRUE;
1280
1281 /*
1282 * Step 2
1283 *
1284 * Update nsw parameters from swap_async_max sysctl values.
1285 * Do not let the sysop crash the machine with bogus numbers.
1286 */
1223 if (object->type != OBJT_SWAP)
1224 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1225
1226 if (curproc != pageproc)
1227 sync = TRUE;
1228
1229 /*
1230 * Step 2
1231 *
1232 * Update nsw parameters from swap_async_max sysctl values.
1233 * Do not let the sysop crash the machine with bogus numbers.
1234 */
1287
1288 mtx_lock(&pbuf_mtx);
1289 if (swap_async_max != nsw_wcount_async_max) {
1290 int n;
1291 int s;
1292
1293 /*
1294 * limit range
1295 */

--- 21 unchanged lines hidden (view full) ---

1317
1318 /*
1319 * Step 3
1320 *
1321 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1322 * The page is left dirty until the pageout operation completes
1323 * successfully.
1324 */
1235 mtx_lock(&pbuf_mtx);
1236 if (swap_async_max != nsw_wcount_async_max) {
1237 int n;
1238 int s;
1239
1240 /*
1241 * limit range
1242 */

--- 21 unchanged lines hidden (view full) ---

1264
1265 /*
1266 * Step 3
1267 *
1268 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1269 * The page is left dirty until the pageout operation completes
1270 * successfully.
1271 */
1325
1326 for (i = 0; i < count; i += n) {
1327 int s;
1328 int j;
1329 struct buf *bp;
1330 daddr_t blk;
1331
1332 /*
1333 * Maximum I/O size is limited by a number of factors.
1334 */
1272 for (i = 0; i < count; i += n) {
1273 int s;
1274 int j;
1275 struct buf *bp;
1276 daddr_t blk;
1277
1278 /*
1279 * Maximum I/O size is limited by a number of factors.
1280 */
1335
1336 n = min(BLIST_MAX_ALLOC, count - i);
1337 n = min(n, nsw_cluster_max);
1338
1339 s = splvm();
1340
1341 /*
1342 * Get biggest block of swap we can. If we fail, fall
1343 * back and try to allocate a smaller block. Don't go

--- 25 unchanged lines hidden (view full) ---

1369 }
1370
1371 /*
1372 * All I/O parameters have been satisfied, build the I/O
1373 * request and assign the swap space.
1374 *
1375 * NOTE: B_PAGING is set by pbgetvp()
1376 */
1281 n = min(BLIST_MAX_ALLOC, count - i);
1282 n = min(n, nsw_cluster_max);
1283
1284 s = splvm();
1285
1286 /*
1287 * Get biggest block of swap we can. If we fail, fall
1288 * back and try to allocate a smaller block. Don't go

--- 25 unchanged lines hidden (view full) ---

1314 }
1315
1316 /*
1317 * All I/O parameters have been satisfied, build the I/O
1318 * request and assign the swap space.
1319 *
1320 * NOTE: B_PAGING is set by pbgetvp()
1321 */
1377
1378 if (sync == TRUE) {
1379 bp = getpbuf(&nsw_wcount_sync);
1380 } else {
1381 bp = getpbuf(&nsw_wcount_async);
1382 bp->b_flags = B_ASYNC;
1383 }
1384 bp->b_iocmd = BIO_WRITE;
1385 bp->b_spc = NULL; /* not used, but NULL-out anyway */

--- 35 unchanged lines hidden (view full) ---

1421
1422 splx(s);
1423
1424 /*
1425 * asynchronous
1426 *
1427 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1428 */
1322 if (sync == TRUE) {
1323 bp = getpbuf(&nsw_wcount_sync);
1324 } else {
1325 bp = getpbuf(&nsw_wcount_async);
1326 bp->b_flags = B_ASYNC;
1327 }
1328 bp->b_iocmd = BIO_WRITE;
1329 bp->b_spc = NULL; /* not used, but NULL-out anyway */

--- 35 unchanged lines hidden (view full) ---

1365
1366 splx(s);
1367
1368 /*
1369 * asynchronous
1370 *
1371 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1372 */
1429
1430 if (sync == FALSE) {
1431 bp->b_iodone = swp_pager_async_iodone;
1432 BUF_KERNPROC(bp);
1433 BUF_STRATEGY(bp);
1434
1435 for (j = 0; j < n; ++j)
1436 rtvals[i+j] = VM_PAGER_PEND;
1437 /* restart outter loop */
1438 continue;
1439 }
1440
1441 /*
1442 * synchronous
1443 *
1444 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1445 */
1373 if (sync == FALSE) {
1374 bp->b_iodone = swp_pager_async_iodone;
1375 BUF_KERNPROC(bp);
1376 BUF_STRATEGY(bp);
1377
1378 for (j = 0; j < n; ++j)
1379 rtvals[i+j] = VM_PAGER_PEND;
1380 /* restart outter loop */
1381 continue;
1382 }
1383
1384 /*
1385 * synchronous
1386 *
1387 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1388 */
1446
1447 bp->b_iodone = swp_pager_sync_iodone;
1448 BUF_STRATEGY(bp);
1449
1450 /*
1451 * Wait for the sync I/O to complete, then update rtvals.
1452 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1453 * our async completion routine at the end, thus avoiding a
1454 * double-free.
1455 */
1456 s = splbio();
1389 bp->b_iodone = swp_pager_sync_iodone;
1390 BUF_STRATEGY(bp);
1391
1392 /*
1393 * Wait for the sync I/O to complete, then update rtvals.
1394 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1395 * our async completion routine at the end, thus avoiding a
1396 * double-free.
1397 */
1398 s = splbio();
1457
1458 while ((bp->b_flags & B_DONE) == 0) {
1459 tsleep(bp, PVM, "swwrt", 0);
1460 }
1399 while ((bp->b_flags & B_DONE) == 0) {
1400 tsleep(bp, PVM, "swwrt", 0);
1401 }
1461
1462 for (j = 0; j < n; ++j)
1463 rtvals[i+j] = VM_PAGER_PEND;
1402 for (j = 0; j < n; ++j)
1403 rtvals[i+j] = VM_PAGER_PEND;
1464
1465 /*
1466 * Now that we are through with the bp, we can call the
1467 * normal async completion, which frees everything up.
1468 */
1404 /*
1405 * Now that we are through with the bp, we can call the
1406 * normal async completion, which frees everything up.
1407 */
1469
1470 swp_pager_async_iodone(bp);
1471 splx(s);
1472 }
1473}
1474
1475/*
1476 * swap_pager_sync_iodone:
1477 *
1478 * Completion routine for synchronous reads and writes from/to swap.
1479 * We just mark the bp is complete and wake up anyone waiting on it.
1480 *
1481 * This routine may not block. This routine is called at splbio() or better.
1482 */
1408 swp_pager_async_iodone(bp);
1409 splx(s);
1410 }
1411}
1412
1413/*
1414 * swap_pager_sync_iodone:
1415 *
1416 * Completion routine for synchronous reads and writes from/to swap.
1417 * We just mark the bp is complete and wake up anyone waiting on it.
1418 *
1419 * This routine may not block. This routine is called at splbio() or better.
1420 */
1483
1484static void
1485swp_pager_sync_iodone(bp)
1486 struct buf *bp;
1487{
1488 bp->b_flags |= B_DONE;
1489 bp->b_flags &= ~B_ASYNC;
1490 wakeup(bp);
1491}

--- 11 unchanged lines hidden (view full) ---

1503 * because we marked them all VM_PAGER_PEND on return from putpages ).
1504 *
1505 * This routine may not block.
1506 * This routine is called at splbio() or better
1507 *
1508 * We up ourselves to splvm() as required for various vm_page related
1509 * calls.
1510 */
1421static void
1422swp_pager_sync_iodone(bp)
1423 struct buf *bp;
1424{
1425 bp->b_flags |= B_DONE;
1426 bp->b_flags &= ~B_ASYNC;
1427 wakeup(bp);
1428}

--- 11 unchanged lines hidden (view full) ---

1440 * because we marked them all VM_PAGER_PEND on return from putpages ).
1441 *
1442 * This routine may not block.
1443 * This routine is called at splbio() or better
1444 *
1445 * We up ourselves to splvm() as required for various vm_page related
1446 * calls.
1447 */
1511
1512static void
1513swp_pager_async_iodone(bp)
1514 struct buf *bp;
1515{
1516 int s;
1517 int i;
1518 vm_object_t object = NULL;
1519
1520 GIANT_REQUIRED;
1448static void
1449swp_pager_async_iodone(bp)
1450 struct buf *bp;
1451{
1452 int s;
1453 int i;
1454 vm_object_t object = NULL;
1455
1456 GIANT_REQUIRED;
1521
1522 bp->b_flags |= B_DONE;
1523
1524 /*
1525 * report error
1526 */
1457 bp->b_flags |= B_DONE;
1458
1459 /*
1460 * report error
1461 */
1527
1528 if (bp->b_ioflags & BIO_ERROR) {
1529 printf(
1530 "swap_pager: I/O error - %s failed; blkno %ld,"
1531 "size %ld, error %d\n",
1532 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1533 (long)bp->b_blkno,
1534 (long)bp->b_bcount,
1535 bp->b_error
1536 );
1537 }
1538
1539 /*
1540 * set object, raise to splvm().
1541 */
1462 if (bp->b_ioflags & BIO_ERROR) {
1463 printf(
1464 "swap_pager: I/O error - %s failed; blkno %ld,"
1465 "size %ld, error %d\n",
1466 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1467 (long)bp->b_blkno,
1468 (long)bp->b_bcount,
1469 bp->b_error
1470 );
1471 }
1472
1473 /*
1474 * set object, raise to splvm().
1475 */
1542
1543 if (bp->b_npages)
1544 object = bp->b_pages[0]->object;
1545 s = splvm();
1546
1547 /*
1548 * remove the mapping for kernel virtual
1549 */
1550 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1551
1552 /*
1553 * cleanup pages. If an error occurs writing to swap, we are in
1554 * very serious trouble. If it happens to be a disk error, though,
1555 * we may be able to recover by reassigning the swap later on. So
1556 * in this case we remove the m->swapblk assignment for the page
1557 * but do not free it in the rlist. The errornous block(s) are thus
1558 * never reallocated as swap. Redirty the page and continue.
1559 */
1476 if (bp->b_npages)
1477 object = bp->b_pages[0]->object;
1478 s = splvm();
1479
1480 /*
1481 * remove the mapping for kernel virtual
1482 */
1483 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1484
1485 /*
1486 * cleanup pages. If an error occurs writing to swap, we are in
1487 * very serious trouble. If it happens to be a disk error, though,
1488 * we may be able to recover by reassigning the swap later on. So
1489 * in this case we remove the m->swapblk assignment for the page
1490 * but do not free it in the rlist. The errornous block(s) are thus
1491 * never reallocated as swap. Redirty the page and continue.
1492 */
1560
1561 for (i = 0; i < bp->b_npages; ++i) {
1562 vm_page_t m = bp->b_pages[i];
1563
1564 vm_page_flag_clear(m, PG_SWAPINPROG);
1565
1566 if (bp->b_ioflags & BIO_ERROR) {
1567 /*
1568 * If an error occurs I'd love to throw the swapblk
1569 * away without freeing it back to swapspace, so it
1570 * can never be used again. But I can't from an
1571 * interrupt.
1572 */
1493 for (i = 0; i < bp->b_npages; ++i) {
1494 vm_page_t m = bp->b_pages[i];
1495
1496 vm_page_flag_clear(m, PG_SWAPINPROG);
1497
1498 if (bp->b_ioflags & BIO_ERROR) {
1499 /*
1500 * If an error occurs I'd love to throw the swapblk
1501 * away without freeing it back to swapspace, so it
1502 * can never be used again. But I can't from an
1503 * interrupt.
1504 */
1573
1574 if (bp->b_iocmd == BIO_READ) {
1575 /*
1576 * When reading, reqpage needs to stay
1577 * locked for the parent, but all other
1578 * pages can be freed. We still want to
1579 * wakeup the parent waiting on the page,
1580 * though. ( also: pg_reqpage can be -1 and
1581 * not match anything ).

--- 6 unchanged lines hidden (view full) ---

1588 * be overridden by the original caller of
1589 * getpages so don't play cute tricks here.
1590 *
1591 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1592 * AS THIS MESSES WITH object->memq, and it is
1593 * not legal to mess with object->memq from an
1594 * interrupt.
1595 */
1505 if (bp->b_iocmd == BIO_READ) {
1506 /*
1507 * When reading, reqpage needs to stay
1508 * locked for the parent, but all other
1509 * pages can be freed. We still want to
1510 * wakeup the parent waiting on the page,
1511 * though. ( also: pg_reqpage can be -1 and
1512 * not match anything ).

--- 6 unchanged lines hidden (view full) ---

1519 * be overridden by the original caller of
1520 * getpages so don't play cute tricks here.
1521 *
1522 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1523 * AS THIS MESSES WITH object->memq, and it is
1524 * not legal to mess with object->memq from an
1525 * interrupt.
1526 */
1596
1597 m->valid = 0;
1598 vm_page_flag_clear(m, PG_ZERO);
1527 m->valid = 0;
1528 vm_page_flag_clear(m, PG_ZERO);
1599
1600 if (i != bp->b_pager.pg_reqpage)
1601 vm_page_free(m);
1602 else
1603 vm_page_flash(m);
1604 /*
1605 * If i == bp->b_pager.pg_reqpage, do not wake
1606 * the page up. The caller needs to.
1607 */

--- 26 unchanged lines hidden (view full) ---

1634 * If not the requested page then deactivate it.
1635 *
1636 * Note that the requested page, reqpage, is left
1637 * busied, but we still have to wake it up. The
1638 * other pages are released (unbusied) by
1639 * vm_page_wakeup(). We do not set reqpage's
1640 * valid bits here, it is up to the caller.
1641 */
1529 if (i != bp->b_pager.pg_reqpage)
1530 vm_page_free(m);
1531 else
1532 vm_page_flash(m);
1533 /*
1534 * If i == bp->b_pager.pg_reqpage, do not wake
1535 * the page up. The caller needs to.
1536 */

--- 26 unchanged lines hidden (view full) ---

1563 * If not the requested page then deactivate it.
1564 *
1565 * Note that the requested page, reqpage, is left
1566 * busied, but we still have to wake it up. The
1567 * other pages are released (unbusied) by
1568 * vm_page_wakeup(). We do not set reqpage's
1569 * valid bits here, it is up to the caller.
1570 */
1642
1643 pmap_clear_modify(m);
1644 m->valid = VM_PAGE_BITS_ALL;
1645 vm_page_undirty(m);
1646 vm_page_flag_clear(m, PG_ZERO);
1647
1648 /*
1649 * We have to wake specifically requested pages
1650 * up too because we cleared PG_SWAPINPROG and

--- 21 unchanged lines hidden (view full) ---

1672 vm_page_protect(m, VM_PROT_READ);
1673 }
1674 }
1675
1676 /*
1677 * adjust pip. NOTE: the original parent may still have its own
1678 * pip refs on the object.
1679 */
1571 pmap_clear_modify(m);
1572 m->valid = VM_PAGE_BITS_ALL;
1573 vm_page_undirty(m);
1574 vm_page_flag_clear(m, PG_ZERO);
1575
1576 /*
1577 * We have to wake specifically requested pages
1578 * up too because we cleared PG_SWAPINPROG and

--- 21 unchanged lines hidden (view full) ---

1600 vm_page_protect(m, VM_PROT_READ);
1601 }
1602 }
1603
1604 /*
1605 * adjust pip. NOTE: the original parent may still have its own
1606 * pip refs on the object.
1607 */
1680
1681 if (object)
1682 vm_object_pip_wakeupn(object, bp->b_npages);
1683
1684 /*
1685 * release the physical I/O buffer
1686 */
1608 if (object)
1609 vm_object_pip_wakeupn(object, bp->b_npages);
1610
1611 /*
1612 * release the physical I/O buffer
1613 */
1687
1688 relpbuf(
1689 bp,
1690 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1691 ((bp->b_flags & B_ASYNC) ?
1692 &nsw_wcount_async :
1693 &nsw_wcount_sync
1694 )
1695 )

--- 20 unchanged lines hidden (view full) ---

1716 *
1717 * This is an inline helper function which hashes the swapblk given
1718 * the object and page index. It returns a pointer to a pointer
1719 * to the object, or a pointer to a NULL pointer if it could not
1720 * find a swapblk.
1721 *
1722 * This routine must be called at splvm().
1723 */
1614 relpbuf(
1615 bp,
1616 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1617 ((bp->b_flags & B_ASYNC) ?
1618 &nsw_wcount_async :
1619 &nsw_wcount_sync
1620 )
1621 )

--- 20 unchanged lines hidden (view full) ---

1642 *
1643 * This is an inline helper function which hashes the swapblk given
1644 * the object and page index. It returns a pointer to a pointer
1645 * to the object, or a pointer to a NULL pointer if it could not
1646 * find a swapblk.
1647 *
1648 * This routine must be called at splvm().
1649 */
1724
1725static __inline struct swblock **
1726swp_pager_hash(vm_object_t object, vm_pindex_t index)
1727{
1728 struct swblock **pswap;
1729 struct swblock *swap;
1730
1731 index &= ~SWAP_META_MASK;
1732 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
1650static __inline struct swblock **
1651swp_pager_hash(vm_object_t object, vm_pindex_t index)
1652{
1653 struct swblock **pswap;
1654 struct swblock *swap;
1655
1656 index &= ~SWAP_META_MASK;
1657 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
1733
1734 while ((swap = *pswap) != NULL) {
1735 if (swap->swb_object == object &&
1736 swap->swb_index == index
1737 ) {
1738 break;
1739 }
1740 pswap = &swap->swb_hnext;
1741 }
1658 while ((swap = *pswap) != NULL) {
1659 if (swap->swb_object == object &&
1660 swap->swb_index == index
1661 ) {
1662 break;
1663 }
1664 pswap = &swap->swb_hnext;
1665 }
1742 return(pswap);
1666 return (pswap);
1743}
1744
1745/*
1746 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1747 *
1748 * We first convert the object to a swap object if it is a default
1749 * object.
1750 *
1751 * The specified swapblk is added to the object's swap metadata. If
1752 * the swapblk is not valid, it is freed instead. Any previously
1753 * assigned swapblk is freed.
1754 *
1755 * This routine must be called at splvm(), except when used to convert
1756 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1757 */
1667}
1668
1669/*
1670 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1671 *
1672 * We first convert the object to a swap object if it is a default
1673 * object.
1674 *
1675 * The specified swapblk is added to the object's swap metadata. If
1676 * the swapblk is not valid, it is freed instead. Any previously
1677 * assigned swapblk is freed.
1678 *
1679 * This routine must be called at splvm(), except when used to convert
1680 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1681 */
1758
1759static void
1760swp_pager_meta_build(
1761 vm_object_t object,
1762 vm_pindex_t index,
1763 daddr_t swapblk
1764) {
1765 struct swblock *swap;
1766 struct swblock **pswap;
1767
1768 GIANT_REQUIRED;
1769 /*
1770 * Convert default object to swap object if necessary
1771 */
1682static void
1683swp_pager_meta_build(
1684 vm_object_t object,
1685 vm_pindex_t index,
1686 daddr_t swapblk
1687) {
1688 struct swblock *swap;
1689 struct swblock **pswap;
1690
1691 GIANT_REQUIRED;
1692 /*
1693 * Convert default object to swap object if necessary
1694 */
1772
1773 if (object->type != OBJT_SWAP) {
1774 object->type = OBJT_SWAP;
1775 object->un_pager.swp.swp_bcount = 0;
1776
1777 mtx_lock(&sw_alloc_mtx);
1778 if (object->handle != NULL) {
1779 TAILQ_INSERT_TAIL(
1780 NOBJLIST(object->handle),

--- 10 unchanged lines hidden (view full) ---

1791 mtx_unlock(&sw_alloc_mtx);
1792 }
1793
1794 /*
1795 * Locate hash entry. If not found create, but if we aren't adding
1796 * anything just return. If we run out of space in the map we wait
1797 * and, since the hash table may have changed, retry.
1798 */
1695 if (object->type != OBJT_SWAP) {
1696 object->type = OBJT_SWAP;
1697 object->un_pager.swp.swp_bcount = 0;
1698
1699 mtx_lock(&sw_alloc_mtx);
1700 if (object->handle != NULL) {
1701 TAILQ_INSERT_TAIL(
1702 NOBJLIST(object->handle),

--- 10 unchanged lines hidden (view full) ---

1713 mtx_unlock(&sw_alloc_mtx);
1714 }
1715
1716 /*
1717 * Locate hash entry. If not found create, but if we aren't adding
1718 * anything just return. If we run out of space in the map we wait
1719 * and, since the hash table may have changed, retry.
1720 */
1799
1800retry:
1801 pswap = swp_pager_hash(object, index);
1802
1803 if ((swap = *pswap) == NULL) {
1804 int i;
1805
1806 if (swapblk == SWAPBLK_NONE)
1807 return;

--- 12 unchanged lines hidden (view full) ---

1820
1821 for (i = 0; i < SWAP_META_PAGES; ++i)
1822 swap->swb_pages[i] = SWAPBLK_NONE;
1823 }
1824
1825 /*
1826 * Delete prior contents of metadata
1827 */
1721retry:
1722 pswap = swp_pager_hash(object, index);
1723
1724 if ((swap = *pswap) == NULL) {
1725 int i;
1726
1727 if (swapblk == SWAPBLK_NONE)
1728 return;

--- 12 unchanged lines hidden (view full) ---

1741
1742 for (i = 0; i < SWAP_META_PAGES; ++i)
1743 swap->swb_pages[i] = SWAPBLK_NONE;
1744 }
1745
1746 /*
1747 * Delete prior contents of metadata
1748 */
1828
1829 index &= SWAP_META_MASK;
1830
1831 if (swap->swb_pages[index] != SWAPBLK_NONE) {
1832 swp_pager_freeswapspace(swap->swb_pages[index], 1);
1833 --swap->swb_count;
1834 }
1835
1836 /*
1837 * Enter block into metadata
1838 */
1749 index &= SWAP_META_MASK;
1750
1751 if (swap->swb_pages[index] != SWAPBLK_NONE) {
1752 swp_pager_freeswapspace(swap->swb_pages[index], 1);
1753 --swap->swb_count;
1754 }
1755
1756 /*
1757 * Enter block into metadata
1758 */
1839
1840 swap->swb_pages[index] = swapblk;
1841 if (swapblk != SWAPBLK_NONE)
1842 ++swap->swb_count;
1843}
1844
1845/*
1846 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1847 *
1848 * The requested range of blocks is freed, with any associated swap
1849 * returned to the swap bitmap.
1850 *
1851 * This routine will free swap metadata structures as they are cleaned
1852 * out. This routine does *NOT* operate on swap metadata associated
1853 * with resident pages.
1854 *
1855 * This routine must be called at splvm()
1856 */
1759 swap->swb_pages[index] = swapblk;
1760 if (swapblk != SWAPBLK_NONE)
1761 ++swap->swb_count;
1762}
1763
1764/*
1765 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1766 *
1767 * The requested range of blocks is freed, with any associated swap
1768 * returned to the swap bitmap.
1769 *
1770 * This routine will free swap metadata structures as they are cleaned
1771 * out. This routine does *NOT* operate on swap metadata associated
1772 * with resident pages.
1773 *
1774 * This routine must be called at splvm()
1775 */
1857
1858static void
1859swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1860{
1861 GIANT_REQUIRED;
1862
1863 if (object->type != OBJT_SWAP)
1864 return;
1865

--- 29 unchanged lines hidden (view full) ---

1895/*
1896 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1897 *
1898 * This routine locates and destroys all swap metadata associated with
1899 * an object.
1900 *
1901 * This routine must be called at splvm()
1902 */
1776static void
1777swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1778{
1779 GIANT_REQUIRED;
1780
1781 if (object->type != OBJT_SWAP)
1782 return;
1783

--- 29 unchanged lines hidden (view full) ---

1813/*
1814 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1815 *
1816 * This routine locates and destroys all swap metadata associated with
1817 * an object.
1818 *
1819 * This routine must be called at splvm()
1820 */
1903
1904static void
1905swp_pager_meta_free_all(vm_object_t object)
1906{
1907 daddr_t index = 0;
1908
1909 GIANT_REQUIRED;
1910
1911 if (object->type != OBJT_SWAP)

--- 43 unchanged lines hidden (view full) ---

1955 * have to wait until paging is complete but otherwise can act on the
1956 * busy page.
1957 *
1958 * This routine must be called at splvm().
1959 *
1960 * SWM_FREE remove and free swap block from metadata
1961 * SWM_POP remove from meta data but do not free.. pop it out
1962 */
1821static void
1822swp_pager_meta_free_all(vm_object_t object)
1823{
1824 daddr_t index = 0;
1825
1826 GIANT_REQUIRED;
1827
1828 if (object->type != OBJT_SWAP)

--- 43 unchanged lines hidden (view full) ---

1872 * have to wait until paging is complete but otherwise can act on the
1873 * busy page.
1874 *
1875 * This routine must be called at splvm().
1876 *
1877 * SWM_FREE remove and free swap block from metadata
1878 * SWM_POP remove from meta data but do not free.. pop it out
1879 */
1963
1964static daddr_t
1965swp_pager_meta_ctl(
1966 vm_object_t object,
1967 vm_pindex_t index,
1968 int flags
1969) {
1970 struct swblock **pswap;
1971 struct swblock *swap;
1972 daddr_t r1;
1973
1974 GIANT_REQUIRED;
1975 /*
1976 * The meta data only exists of the object is OBJT_SWAP
1977 * and even then might not be allocated yet.
1978 */
1880static daddr_t
1881swp_pager_meta_ctl(
1882 vm_object_t object,
1883 vm_pindex_t index,
1884 int flags
1885) {
1886 struct swblock **pswap;
1887 struct swblock *swap;
1888 daddr_t r1;
1889
1890 GIANT_REQUIRED;
1891 /*
1892 * The meta data only exists of the object is OBJT_SWAP
1893 * and even then might not be allocated yet.
1894 */
1979
1980 if (object->type != OBJT_SWAP)
1895 if (object->type != OBJT_SWAP)
1981 return(SWAPBLK_NONE);
1896 return (SWAPBLK_NONE);
1982
1983 r1 = SWAPBLK_NONE;
1984 pswap = swp_pager_hash(object, index);
1985
1986 if ((swap = *pswap) != NULL) {
1987 index &= SWAP_META_MASK;
1988 r1 = swap->swb_pages[index];
1989

--- 7 unchanged lines hidden (view full) ---

1997 if (--swap->swb_count == 0) {
1998 *pswap = swap->swb_hnext;
1999 zfree(swap_zone, swap);
2000 --object->un_pager.swp.swp_bcount;
2001 }
2002 }
2003 }
2004 }
1897
1898 r1 = SWAPBLK_NONE;
1899 pswap = swp_pager_hash(object, index);
1900
1901 if ((swap = *pswap) != NULL) {
1902 index &= SWAP_META_MASK;
1903 r1 = swap->swb_pages[index];
1904

--- 7 unchanged lines hidden (view full) ---

1912 if (--swap->swb_count == 0) {
1913 *pswap = swap->swb_hnext;
1914 zfree(swap_zone, swap);
1915 --object->un_pager.swp.swp_bcount;
1916 }
1917 }
1918 }
1919 }
2005 return(r1);
1920 return (r1);
2006}
2007
2008/********************************************************
2009 * CHAINING FUNCTIONS *
2010 ********************************************************
2011 *
2012 * These functions support recursion of I/O operations
2013 * on bp's, typically by chaining one or more 'child' bp's
2014 * to the parent. Synchronous, asynchronous, and semi-synchronous
2015 * chaining is possible.
2016 */
2017
2018/*
2019 * vm_pager_chain_iodone:
2020 *
2021 * io completion routine for child bp. Currently we fudge a bit
2022 * on dealing with b_resid. Since users of these routines may issue
2023 * multiple children simultaneously, sequencing of the error can be lost.
2024 */
1921}
1922
1923/********************************************************
1924 * CHAINING FUNCTIONS *
1925 ********************************************************
1926 *
1927 * These functions support recursion of I/O operations
1928 * on bp's, typically by chaining one or more 'child' bp's
1929 * to the parent. Synchronous, asynchronous, and semi-synchronous
1930 * chaining is possible.
1931 */
1932
1933/*
1934 * vm_pager_chain_iodone:
1935 *
1936 * io completion routine for child bp. Currently we fudge a bit
1937 * on dealing with b_resid. Since users of these routines may issue
1938 * multiple children simultaneously, sequencing of the error can be lost.
1939 */
2025
2026static void
2027vm_pager_chain_iodone(struct buf *nbp)
2028{
2029 struct bio *bp;
2030 u_int *count;
2031
2032 bp = nbp->b_caller1;
2033 count = (u_int *)&(bp->bio_driver1);

--- 21 unchanged lines hidden (view full) ---

2055
2056/*
2057 * getchainbuf:
2058 *
2059 * Obtain a physical buffer and chain it to its parent buffer. When
2060 * I/O completes, the parent buffer will be B_SIGNAL'd. Errors are
2061 * automatically propagated to the parent
2062 */
1940static void
1941vm_pager_chain_iodone(struct buf *nbp)
1942{
1943 struct bio *bp;
1944 u_int *count;
1945
1946 bp = nbp->b_caller1;
1947 count = (u_int *)&(bp->bio_driver1);

--- 21 unchanged lines hidden (view full) ---

1969
1970/*
1971 * getchainbuf:
1972 *
1973 * Obtain a physical buffer and chain it to its parent buffer. When
1974 * I/O completes, the parent buffer will be B_SIGNAL'd. Errors are
1975 * automatically propagated to the parent
1976 */
2063
2064struct buf *
2065getchainbuf(struct bio *bp, struct vnode *vp, int flags)
2066{
2067 struct buf *nbp;
2068 u_int *count;
2069
2070 GIANT_REQUIRED;
2071 nbp = getpbuf(NULL);

--- 9 unchanged lines hidden (view full) ---

2081 nbp->b_ioflags = 0;
2082 nbp->b_flags = flags;
2083 nbp->b_rcred = crhold(thread0.td_ucred);
2084 nbp->b_wcred = crhold(thread0.td_ucred);
2085 nbp->b_iodone = vm_pager_chain_iodone;
2086
2087 if (vp)
2088 pbgetvp(vp, nbp);
1977struct buf *
1978getchainbuf(struct bio *bp, struct vnode *vp, int flags)
1979{
1980 struct buf *nbp;
1981 u_int *count;
1982
1983 GIANT_REQUIRED;
1984 nbp = getpbuf(NULL);

--- 9 unchanged lines hidden (view full) ---

1994 nbp->b_ioflags = 0;
1995 nbp->b_flags = flags;
1996 nbp->b_rcred = crhold(thread0.td_ucred);
1997 nbp->b_wcred = crhold(thread0.td_ucred);
1998 nbp->b_iodone = vm_pager_chain_iodone;
1999
2000 if (vp)
2001 pbgetvp(vp, nbp);
2089 return(nbp);
2002 return (nbp);
2090}
2091
2092void
2093flushchainbuf(struct buf *nbp)
2094{
2095 GIANT_REQUIRED;
2096 if (nbp->b_bcount) {
2097 nbp->b_bufsize = nbp->b_bcount;

--- 32 unchanged lines hidden --- 		2003}
2004
2005void
2006flushchainbuf(struct buf *nbp)
2007{
2008 GIANT_REQUIRED;
2009 if (nbp->b_bcount) {
2010 nbp->b_bufsize = nbp->b_bcount;

--- 32 unchanged lines hidden ---