158 struct vnode *sw_vp;
159 void *sw_id;
160 swblk_t sw_first;
161 swblk_t sw_end;
162 struct blist *sw_blist;
163 TAILQ_ENTRY(swdevt) sw_list;
164 sw_strategy_t *sw_strategy;
165 sw_close_t *sw_close;
166};
167
168#define SW_CLOSING 0x04
169
170struct swblock {
171 struct swblock *swb_hnext;
172 vm_object_t swb_object;
173 vm_pindex_t swb_index;
174 int swb_count;
175 daddr_t swb_pages[SWAP_META_PAGES];
176};
177
178static struct mtx sw_dev_mtx;
179static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
180static struct swdevt *swdevhd; /* Allocate from here next */
181static int nswapdev; /* Number of swap devices */
182int swap_pager_avail;
183static int swdev_syscall_active = 0; /* serialize swap(on|off) */
184
185static void swapdev_strategy(struct buf *, struct swdevt *sw);
186
187#define SWM_FREE 0x02 /* free, period */
188#define SWM_POP 0x04 /* pop out */
189
190int swap_pager_full = 2; /* swap space exhaustion (task killing) */
191static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
192static int nsw_rcount; /* free read buffers */
193static int nsw_wcount_sync; /* limit write buffers / synchronous */
194static int nsw_wcount_async; /* limit write buffers / asynchronous */
195static int nsw_wcount_async_max;/* assigned maximum */
196static int nsw_cluster_max; /* maximum VOP I/O allowed */
197
198static struct swblock **swhash;
199static int swhash_mask;
200static struct mtx swhash_mtx;
201
202static int swap_async_max = 4; /* maximum in-progress async I/O's */
203static struct sx sw_alloc_sx;
204
205
206SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
207 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
208
209/*
210 * "named" and "unnamed" anon region objects. Try to reduce the overhead
211 * of searching a named list by hashing it just a little.
212 */
213
214#define NOBJLISTS 8
215
216#define NOBJLIST(handle) \
217 (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
218
219static struct mtx sw_alloc_mtx; /* protect list manipulation */
220static struct pagerlst swap_pager_object_list[NOBJLISTS];
221static uma_zone_t swap_zone;
222
223/*
224 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure
225 * calls hooked from other parts of the VM system and do not appear here.
226 * (see vm/swap_pager.h).
227 */
228static vm_object_t
229 swap_pager_alloc(void *handle, vm_ooffset_t size,
230 vm_prot_t prot, vm_ooffset_t offset);
231static void swap_pager_dealloc(vm_object_t object);
232static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
233static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
234static boolean_t
235 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
236static void swap_pager_init(void);
237static void swap_pager_unswapped(vm_page_t);
238static void swap_pager_swapoff(struct swdevt *sp, int *sw_used);
239
240struct pagerops swappagerops = {
241 .pgo_init = swap_pager_init, /* early system initialization of pager */
242 .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */
243 .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */
244 .pgo_getpages = swap_pager_getpages, /* pagein */
245 .pgo_putpages = swap_pager_putpages, /* pageout */
246 .pgo_haspage = swap_pager_haspage, /* get backing store status for page */
247 .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */
248};
249
250/*
251 * dmmax is in page-sized chunks with the new swap system. It was
252 * dev-bsized chunks in the old. dmmax is always a power of 2.
253 *
254 * swap_*() routines are externally accessible. swp_*() routines are
255 * internal.
256 */
257static int dmmax;
258static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */
259static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */
260
261SYSCTL_INT(_vm, OID_AUTO, dmmax,
262 CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
263
264static void swp_sizecheck(void);
265static void swp_pager_async_iodone(struct buf *bp);
266static int swapongeom(struct thread *, struct vnode *);
267static int swaponvp(struct thread *, struct vnode *, u_long);
268
269/*
270 * Swap bitmap functions
271 */
272static void swp_pager_freeswapspace(daddr_t blk, int npages);
273static daddr_t swp_pager_getswapspace(int npages);
274
275/*
276 * Metadata functions
277 */
278static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
279static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
280static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
281static void swp_pager_meta_free_all(vm_object_t);
282static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
283
284/*
285 * SWP_SIZECHECK() - update swap_pager_full indication
286 *
287 * update the swap_pager_almost_full indication and warn when we are
288 * about to run out of swap space, using lowat/hiwat hysteresis.
289 *
290 * Clear swap_pager_full ( task killing ) indication when lowat is met.
291 *
292 * No restrictions on call
293 * This routine may not block.
294 * This routine must be called at splvm()
295 */
296static void
297swp_sizecheck(void)
298{
299
300 if (swap_pager_avail < nswap_lowat) {
301 if (swap_pager_almost_full == 0) {
302 printf("swap_pager: out of swap space\n");
303 swap_pager_almost_full = 1;
304 }
305 } else {
306 swap_pager_full = 0;
307 if (swap_pager_avail > nswap_hiwat)
308 swap_pager_almost_full = 0;
309 }
310}
311
312/*
313 * SWP_PAGER_HASH() - hash swap meta data
314 *
315 * This is an helper function which hashes the swapblk given
316 * the object and page index. It returns a pointer to a pointer
317 * to the object, or a pointer to a NULL pointer if it could not
318 * find a swapblk.
319 *
320 * This routine must be called at splvm().
321 */
322static struct swblock **
323swp_pager_hash(vm_object_t object, vm_pindex_t index)
324{
325 struct swblock **pswap;
326 struct swblock *swap;
327
328 index &= ~(vm_pindex_t)SWAP_META_MASK;
329 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
330 while ((swap = *pswap) != NULL) {
331 if (swap->swb_object == object &&
332 swap->swb_index == index
333 ) {
334 break;
335 }
336 pswap = &swap->swb_hnext;
337 }
338 return (pswap);
339}
340
341/*
342 * SWAP_PAGER_INIT() - initialize the swap pager!
343 *
344 * Expected to be started from system init. NOTE: This code is run
345 * before much else so be careful what you depend on. Most of the VM
346 * system has yet to be initialized at this point.
347 */
348static void
349swap_pager_init(void)
350{
351 /*
352 * Initialize object lists
353 */
354 int i;
355
356 for (i = 0; i < NOBJLISTS; ++i)
357 TAILQ_INIT(&swap_pager_object_list[i]);
358 mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
359 mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
360
361 /*
362 * Device Stripe, in PAGE_SIZE'd blocks
363 */
364 dmmax = SWB_NPAGES * 2;
365}
366
367/*
368 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
369 *
370 * Expected to be started from pageout process once, prior to entering
371 * its main loop.
372 */
373void
374swap_pager_swap_init(void)
375{
376 int n, n2;
377
378 /*
379 * Number of in-transit swap bp operations. Don't
380 * exhaust the pbufs completely. Make sure we
381 * initialize workable values (0 will work for hysteresis
382 * but it isn't very efficient).
383 *
384 * The nsw_cluster_max is constrained by the bp->b_pages[]
385 * array (MAXPHYS/PAGE_SIZE) and our locally defined
386 * MAX_PAGEOUT_CLUSTER. Also be aware that swap ops are
387 * constrained by the swap device interleave stripe size.
388 *
389 * Currently we hardwire nsw_wcount_async to 4. This limit is
390 * designed to prevent other I/O from having high latencies due to
391 * our pageout I/O. The value 4 works well for one or two active swap
392 * devices but is probably a little low if you have more. Even so,
393 * a higher value would probably generate only a limited improvement
394 * with three or four active swap devices since the system does not
395 * typically have to pageout at extreme bandwidths. We will want
396 * at least 2 per swap devices, and 4 is a pretty good value if you
397 * have one NFS swap device due to the command/ack latency over NFS.
398 * So it all works out pretty well.
399 */
400 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
401
402 mtx_lock(&pbuf_mtx);
403 nsw_rcount = (nswbuf + 1) / 2;
404 nsw_wcount_sync = (nswbuf + 3) / 4;
405 nsw_wcount_async = 4;
406 nsw_wcount_async_max = nsw_wcount_async;
407 mtx_unlock(&pbuf_mtx);
408
409 /*
410 * Initialize our zone. Right now I'm just guessing on the number
411 * we need based on the number of pages in the system. Each swblock
412 * can hold 16 pages, so this is probably overkill. This reservation
413 * is typically limited to around 32MB by default.
414 */
415 n = cnt.v_page_count / 2;
416 if (maxswzone && n > maxswzone / sizeof(struct swblock))
417 n = maxswzone / sizeof(struct swblock);
418 n2 = n;
419 swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
420 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
421 do {
422 if (uma_zone_set_obj(swap_zone, NULL, n))
423 break;
424 /*
425 * if the allocation failed, try a zone two thirds the
426 * size of the previous attempt.
427 */
428 n -= ((n + 2) / 3);
429 } while (n > 0);
430 if (swap_zone == NULL)
431 panic("failed to create swap_zone.");
432 if (n2 != n)
433 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
434 n2 = n;
435
436 /*
437 * Initialize our meta-data hash table. The swapper does not need to
438 * be quite as efficient as the VM system, so we do not use an
439 * oversized hash table.
440 *
441 * n: size of hash table, must be power of 2
442 * swhash_mask: hash table index mask
443 */
444 for (n = 1; n < n2 / 8; n *= 2)
445 ;
446 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
447 swhash_mask = n - 1;
448 mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
449}
450
451/*
452 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
453 * its metadata structures.
454 *
455 * This routine is called from the mmap and fork code to create a new
456 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
457 * and then converting it with swp_pager_meta_build().
458 *
459 * This routine may block in vm_object_allocate() and create a named
460 * object lookup race, so we must interlock. We must also run at
461 * splvm() for the object lookup to handle races with interrupts, but
462 * we do not have to maintain splvm() in between the lookup and the
463 * add because (I believe) it is not possible to attempt to create
464 * a new swap object w/handle when a default object with that handle
465 * already exists.
466 *
467 * MPSAFE
468 */
469static vm_object_t
470swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
471 vm_ooffset_t offset)
472{
473 vm_object_t object;
474 vm_pindex_t pindex;
475
476 pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
477
478 if (handle) {
479 mtx_lock(&Giant);
480 /*
481 * Reference existing named region or allocate new one. There
482 * should not be a race here against swp_pager_meta_build()
483 * as called from vm_page_remove() in regards to the lookup
484 * of the handle.
485 */
486 sx_xlock(&sw_alloc_sx);
487 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
488
489 if (object != NULL) {
490 vm_object_reference(object);
491 } else {
492 object = vm_object_allocate(OBJT_DEFAULT, pindex);
493 object->handle = handle;
494
495 VM_OBJECT_LOCK(object);
496 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
497 VM_OBJECT_UNLOCK(object);
498 }
499 sx_xunlock(&sw_alloc_sx);
500 mtx_unlock(&Giant);
501 } else {
502 object = vm_object_allocate(OBJT_DEFAULT, pindex);
503
504 VM_OBJECT_LOCK(object);
505 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
506 VM_OBJECT_UNLOCK(object);
507 }
508 return (object);
509}
510
511/*
512 * SWAP_PAGER_DEALLOC() - remove swap metadata from object
513 *
514 * The swap backing for the object is destroyed. The code is
515 * designed such that we can reinstantiate it later, but this
516 * routine is typically called only when the entire object is
517 * about to be destroyed.
518 *
519 * This routine may block, but no longer does.
520 *
521 * The object must be locked or unreferenceable.
522 */
523static void
524swap_pager_dealloc(vm_object_t object)
525{
526 int s;
527
528 /*
529 * Remove from list right away so lookups will fail if we block for
530 * pageout completion.
531 */
532 if (object->handle != NULL) {
533 mtx_lock(&sw_alloc_mtx);
534 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
535 mtx_unlock(&sw_alloc_mtx);
536 }
537
538 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
539 vm_object_pip_wait(object, "swpdea");
540
541 /*
542 * Free all remaining metadata. We only bother to free it from
543 * the swap meta data. We do not attempt to free swapblk's still
544 * associated with vm_page_t's for this object. We do not care
545 * if paging is still in progress on some objects.
546 */
547 s = splvm();
548 swp_pager_meta_free_all(object);
549 splx(s);
550}
551
552/************************************************************************
553 * SWAP PAGER BITMAP ROUTINES *
554 ************************************************************************/
555
556/*
557 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space
558 *
559 * Allocate swap for the requested number of pages. The starting
560 * swap block number (a page index) is returned or SWAPBLK_NONE
561 * if the allocation failed.
562 *
563 * Also has the side effect of advising that somebody made a mistake
564 * when they configured swap and didn't configure enough.
565 *
566 * Must be called at splvm() to avoid races with bitmap frees from
567 * vm_page_remove() aka swap_pager_page_removed().
568 *
569 * This routine may not block
570 * This routine must be called at splvm().
571 *
572 * We allocate in round-robin fashion from the configured devices.
573 */
574static daddr_t
575swp_pager_getswapspace(int npages)
576{
577 daddr_t blk;
578 struct swdevt *sp;
579 int i;
580
581 blk = SWAPBLK_NONE;
582 mtx_lock(&sw_dev_mtx);
583 sp = swdevhd;
584 for (i = 0; i < nswapdev; i++) {
585 if (sp == NULL)
586 sp = TAILQ_FIRST(&swtailq);
587 if (!(sp->sw_flags & SW_CLOSING)) {
588 blk = blist_alloc(sp->sw_blist, npages);
589 if (blk != SWAPBLK_NONE) {
590 blk += sp->sw_first;
591 sp->sw_used += npages;
592 swap_pager_avail -= npages;
593 swp_sizecheck();
594 swdevhd = TAILQ_NEXT(sp, sw_list);
595 goto done;
596 }
597 }
598 sp = TAILQ_NEXT(sp, sw_list);
599 }
600 if (swap_pager_full != 2) {
601 printf("swap_pager_getswapspace(%d): failed\n", npages);
602 swap_pager_full = 2;
603 swap_pager_almost_full = 1;
604 }
605 swdevhd = NULL;
606done:
607 mtx_unlock(&sw_dev_mtx);
608 return (blk);
609}
610
611static struct swdevt *
612swp_pager_find_dev(daddr_t blk)
613{
614 struct swdevt *sp;
615
616 mtx_lock(&sw_dev_mtx);
617 TAILQ_FOREACH(sp, &swtailq, sw_list) {
618 if (blk >= sp->sw_first && blk < sp->sw_end) {
619 mtx_unlock(&sw_dev_mtx);
620 return (sp);
621 }
622 }
623 panic("Swapdev not found");
624}
625
626static void
627swp_pager_strategy(struct buf *bp)
628{
629 struct swdevt *sp;
630
631 mtx_lock(&sw_dev_mtx);
632 TAILQ_FOREACH(sp, &swtailq, sw_list) {
633 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
634 mtx_unlock(&sw_dev_mtx);
635 sp->sw_strategy(bp, sp);
636 return;
637 }
638 }
639 panic("Swapdev not found");
640}
641
642
643/*
644 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
645 *
646 * This routine returns the specified swap blocks back to the bitmap.
647 *
648 * Note: This routine may not block (it could in the old swap code),
649 * and through the use of the new blist routines it does not block.
650 *
651 * We must be called at splvm() to avoid races with bitmap frees from
652 * vm_page_remove() aka swap_pager_page_removed().
653 *
654 * This routine may not block
655 * This routine must be called at splvm().
656 */
657static void
658swp_pager_freeswapspace(daddr_t blk, int npages)
659{
660 struct swdevt *sp;
661
662 mtx_lock(&sw_dev_mtx);
663 TAILQ_FOREACH(sp, &swtailq, sw_list) {
664 if (blk >= sp->sw_first && blk < sp->sw_end) {
665 sp->sw_used -= npages;
666 /*
667 * If we are attempting to stop swapping on
668 * this device, we don't want to mark any
669 * blocks free lest they be reused.
670 */
671 if ((sp->sw_flags & SW_CLOSING) == 0) {
672 blist_free(sp->sw_blist, blk - sp->sw_first,
673 npages);
674 swap_pager_avail += npages;
675 swp_sizecheck();
676 }
677 mtx_unlock(&sw_dev_mtx);
678 return;
679 }
680 }
681 panic("Swapdev not found");
682}
683
684/*
685 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
686 * range within an object.
687 *
688 * This is a globally accessible routine.
689 *
690 * This routine removes swapblk assignments from swap metadata.
691 *
692 * The external callers of this routine typically have already destroyed
693 * or renamed vm_page_t's associated with this range in the object so
694 * we should be ok.
695 *
696 * This routine may be called at any spl. We up our spl to splvm temporarily
697 * in order to perform the metadata removal.
698 */
699void
700swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
701{
702 int s = splvm();
703
704 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
705 swp_pager_meta_free(object, start, size);
706 splx(s);
707}
708
709/*
710 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
711 *
712 * Assigns swap blocks to the specified range within the object. The
713 * swap blocks are not zerod. Any previous swap assignment is destroyed.
714 *
715 * Returns 0 on success, -1 on failure.
716 */
717int
718swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
719{
720 int s;
721 int n = 0;
722 daddr_t blk = SWAPBLK_NONE;
723 vm_pindex_t beg = start; /* save start index */
724
725 s = splvm();
726 VM_OBJECT_LOCK(object);
727 while (size) {
728 if (n == 0) {
729 n = BLIST_MAX_ALLOC;
730 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
731 n >>= 1;
732 if (n == 0) {
733 swp_pager_meta_free(object, beg, start - beg);
734 VM_OBJECT_UNLOCK(object);
735 splx(s);
736 return (-1);
737 }
738 }
739 }
740 swp_pager_meta_build(object, start, blk);
741 --size;
742 ++start;
743 ++blk;
744 --n;
745 }
746 swp_pager_meta_free(object, start, n);
747 VM_OBJECT_UNLOCK(object);
748 splx(s);
749 return (0);
750}
751
752/*
753 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
754 * and destroy the source.
755 *
756 * Copy any valid swapblks from the source to the destination. In
757 * cases where both the source and destination have a valid swapblk,
758 * we keep the destination's.
759 *
760 * This routine is allowed to block. It may block allocating metadata
761 * indirectly through swp_pager_meta_build() or if paging is still in
762 * progress on the source.
763 *
764 * This routine can be called at any spl
765 *
766 * XXX vm_page_collapse() kinda expects us not to block because we
767 * supposedly do not need to allocate memory, but for the moment we
768 * *may* have to get a little memory from the zone allocator, but
769 * it is taken from the interrupt memory. We should be ok.
770 *
771 * The source object contains no vm_page_t's (which is just as well)
772 *
773 * The source object is of type OBJT_SWAP.
774 *
775 * The source and destination objects must be locked or
776 * inaccessible (XXX are they ?)
777 */
778void
779swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
780 vm_pindex_t offset, int destroysource)
781{
782 vm_pindex_t i;
783 int s;
784
785 VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
786 VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
787
788 s = splvm();
789 /*
790 * If destroysource is set, we remove the source object from the
791 * swap_pager internal queue now.
792 */
793 if (destroysource) {
794 if (srcobject->handle != NULL) {
795 mtx_lock(&sw_alloc_mtx);
796 TAILQ_REMOVE(
797 NOBJLIST(srcobject->handle),
798 srcobject,
799 pager_object_list
800 );
801 mtx_unlock(&sw_alloc_mtx);
802 }
803 }
804
805 /*
806 * transfer source to destination.
807 */
808 for (i = 0; i < dstobject->size; ++i) {
809 daddr_t dstaddr;
810
811 /*
812 * Locate (without changing) the swapblk on the destination,
813 * unless it is invalid in which case free it silently, or
814 * if the destination is a resident page, in which case the
815 * source is thrown away.
816 */
817 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
818
819 if (dstaddr == SWAPBLK_NONE) {
820 /*
821 * Destination has no swapblk and is not resident,
822 * copy source.
823 */
824 daddr_t srcaddr;
825
826 srcaddr = swp_pager_meta_ctl(
827 srcobject,
828 i + offset,
829 SWM_POP
830 );
831
832 if (srcaddr != SWAPBLK_NONE) {
833 /*
834 * swp_pager_meta_build() can sleep.
835 */
836 vm_object_pip_add(srcobject, 1);
837 VM_OBJECT_UNLOCK(srcobject);
838 vm_object_pip_add(dstobject, 1);
839 swp_pager_meta_build(dstobject, i, srcaddr);
840 vm_object_pip_wakeup(dstobject);
841 VM_OBJECT_LOCK(srcobject);
842 vm_object_pip_wakeup(srcobject);
843 }
844 } else {
845 /*
846 * Destination has valid swapblk or it is represented
847 * by a resident page. We destroy the sourceblock.
848 */
849
850 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
851 }
852 }
853
854 /*
855 * Free left over swap blocks in source.
856 *
857 * We have to revert the type to OBJT_DEFAULT so we do not accidently
858 * double-remove the object from the swap queues.
859 */
860 if (destroysource) {
861 swp_pager_meta_free_all(srcobject);
862 /*
863 * Reverting the type is not necessary, the caller is going
864 * to destroy srcobject directly, but I'm doing it here
865 * for consistency since we've removed the object from its
866 * queues.
867 */
868 srcobject->type = OBJT_DEFAULT;
869 }
870 splx(s);
871}
872
873/*
874 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for
875 * the requested page.
876 *
877 * We determine whether good backing store exists for the requested
878 * page and return TRUE if it does, FALSE if it doesn't.
879 *
880 * If TRUE, we also try to determine how much valid, contiguous backing
881 * store exists before and after the requested page within a reasonable
882 * distance. We do not try to restrict it to the swap device stripe
883 * (that is handled in getpages/putpages). It probably isn't worth
884 * doing here.
885 */
886static boolean_t
887swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
888{
889 daddr_t blk0;
890 int s;
891
892 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
893 /*
894 * do we have good backing store at the requested index ?
895 */
896 s = splvm();
897 blk0 = swp_pager_meta_ctl(object, pindex, 0);
898
899 if (blk0 == SWAPBLK_NONE) {
900 splx(s);
901 if (before)
902 *before = 0;
903 if (after)
904 *after = 0;
905 return (FALSE);
906 }
907
908 /*
909 * find backwards-looking contiguous good backing store
910 */
911 if (before != NULL) {
912 int i;
913
914 for (i = 1; i < (SWB_NPAGES/2); ++i) {
915 daddr_t blk;
916
917 if (i > pindex)
918 break;
919 blk = swp_pager_meta_ctl(object, pindex - i, 0);
920 if (blk != blk0 - i)
921 break;
922 }
923 *before = (i - 1);
924 }
925
926 /*
927 * find forward-looking contiguous good backing store
928 */
929 if (after != NULL) {
930 int i;
931
932 for (i = 1; i < (SWB_NPAGES/2); ++i) {
933 daddr_t blk;
934
935 blk = swp_pager_meta_ctl(object, pindex + i, 0);
936 if (blk != blk0 + i)
937 break;
938 }
939 *after = (i - 1);
940 }
941 splx(s);
942 return (TRUE);
943}
944
945/*
946 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
947 *
948 * This removes any associated swap backing store, whether valid or
949 * not, from the page.
950 *
951 * This routine is typically called when a page is made dirty, at
952 * which point any associated swap can be freed. MADV_FREE also
953 * calls us in a special-case situation
954 *
955 * NOTE!!! If the page is clean and the swap was valid, the caller
956 * should make the page dirty before calling this routine. This routine
957 * does NOT change the m->dirty status of the page. Also: MADV_FREE
958 * depends on it.
959 *
960 * This routine may not block
961 * This routine must be called at splvm()
962 */
963static void
964swap_pager_unswapped(vm_page_t m)
965{
966
967 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
968 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
969}
970
971/*
972 * SWAP_PAGER_GETPAGES() - bring pages in from swap
973 *
974 * Attempt to retrieve (m, count) pages from backing store, but make
975 * sure we retrieve at least m[reqpage]. We try to load in as large
976 * a chunk surrounding m[reqpage] as is contiguous in swap and which
977 * belongs to the same object.
978 *
979 * The code is designed for asynchronous operation and
980 * immediate-notification of 'reqpage' but tends not to be
981 * used that way. Please do not optimize-out this algorithmic
982 * feature, I intend to improve on it in the future.
983 *
984 * The parent has a single vm_object_pip_add() reference prior to
985 * calling us and we should return with the same.
986 *
987 * The parent has BUSY'd the pages. We should return with 'm'
988 * left busy, but the others adjusted.
989 */
990static int
991swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
992{
993 struct buf *bp;
994 vm_page_t mreq;
995 int s;
996 int i;
997 int j;
998 daddr_t blk;
999
1000 mreq = m[reqpage];
1001
1002 KASSERT(mreq->object == object,
1003 ("swap_pager_getpages: object mismatch %p/%p",
1004 object, mreq->object));
1005
1006 /*
1007 * Calculate range to retrieve. The pages have already been assigned
1008 * their swapblks. We require a *contiguous* range but we know it to
1009 * not span devices. If we do not supply it, bad things
1010 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1011 * loops are set up such that the case(s) are handled implicitly.
1012 *
1013 * The swp_*() calls must be made at splvm(). vm_page_free() does
1014 * not need to be, but it will go a little faster if it is.
1015 */
1016 s = splvm();
1017 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1018
1019 for (i = reqpage - 1; i >= 0; --i) {
1020 daddr_t iblk;
1021
1022 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1023 if (blk != iblk + (reqpage - i))
1024 break;
1025 }
1026 ++i;
1027
1028 for (j = reqpage + 1; j < count; ++j) {
1029 daddr_t jblk;
1030
1031 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1032 if (blk != jblk - (j - reqpage))
1033 break;
1034 }
1035
1036 /*
1037 * free pages outside our collection range. Note: we never free
1038 * mreq, it must remain busy throughout.
1039 */
1040 vm_page_lock_queues();
1041 {
1042 int k;
1043
1044 for (k = 0; k < i; ++k)
1045 vm_page_free(m[k]);
1046 for (k = j; k < count; ++k)
1047 vm_page_free(m[k]);
1048 }
1049 vm_page_unlock_queues();
1050 splx(s);
1051
1052
1053 /*
1054 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1055 * still busy, but the others unbusied.
1056 */
1057 if (blk == SWAPBLK_NONE)
1058 return (VM_PAGER_FAIL);
1059
1060 /*
1061 * Getpbuf() can sleep.
1062 */
1063 VM_OBJECT_UNLOCK(object);
1064 /*
1065 * Get a swap buffer header to perform the IO
1066 */
1067 bp = getpbuf(&nsw_rcount);
1068 bp->b_flags |= B_PAGING;
1069
1070 /*
1071 * map our page(s) into kva for input
1072 */
1073 pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
1074
1075 bp->b_iocmd = BIO_READ;
1076 bp->b_iodone = swp_pager_async_iodone;
1077 bp->b_rcred = crhold(thread0.td_ucred);
1078 bp->b_wcred = crhold(thread0.td_ucred);
1079 bp->b_blkno = blk - (reqpage - i);
1080 bp->b_bcount = PAGE_SIZE * (j - i);
1081 bp->b_bufsize = PAGE_SIZE * (j - i);
1082 bp->b_pager.pg_reqpage = reqpage - i;
1083
1084 VM_OBJECT_LOCK(object);
1085 vm_page_lock_queues();
1086 {
1087 int k;
1088
1089 for (k = i; k < j; ++k) {
1090 bp->b_pages[k - i] = m[k];
1091 vm_page_flag_set(m[k], PG_SWAPINPROG);
1092 }
1093 }
1094 vm_page_unlock_queues();
1095 VM_OBJECT_UNLOCK(object);
1096 bp->b_npages = j - i;
1097
1098 cnt.v_swapin++;
1099 cnt.v_swappgsin += bp->b_npages;
1100
1101 /*
1102 * We still hold the lock on mreq, and our automatic completion routine
1103 * does not remove it.
1104 */
1105 VM_OBJECT_LOCK(mreq->object);
1106 vm_object_pip_add(mreq->object, bp->b_npages);
1107 VM_OBJECT_UNLOCK(mreq->object);
1108
1109 /*
1110 * perform the I/O. NOTE!!! bp cannot be considered valid after
1111 * this point because we automatically release it on completion.
1112 * Instead, we look at the one page we are interested in which we
1113 * still hold a lock on even through the I/O completion.
1114 *
1115 * The other pages in our m[] array are also released on completion,
1116 * so we cannot assume they are valid anymore either.
1117 *
1118 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1119 */
1120 BUF_KERNPROC(bp);
1121 swp_pager_strategy(bp);
1122
1123 /*
1124 * wait for the page we want to complete. PG_SWAPINPROG is always
1125 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1126 * is set in the meta-data.
1127 */
1128 s = splvm();
1129 vm_page_lock_queues();
1130 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1131 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1132 cnt.v_intrans++;
1133 if (msleep(mreq, &vm_page_queue_mtx, PSWP, "swread", hz*20)) {
1134 printf(
1135 "swap_pager: indefinite wait buffer: device:"
1136 " %s, blkno: %ld, size: %ld\n",
1137 devtoname(bp->b_dev), (long)bp->b_blkno,
1138 bp->b_bcount
1139 );
1140 }
1141 }
1142 vm_page_unlock_queues();
1143 splx(s);
1144
1145 VM_OBJECT_LOCK(mreq->object);
1146 /*
1147 * mreq is left busied after completion, but all the other pages
1148 * are freed. If we had an unrecoverable read error the page will
1149 * not be valid.
1150 */
1151 if (mreq->valid != VM_PAGE_BITS_ALL) {
1152 return (VM_PAGER_ERROR);
1153 } else {
1154 return (VM_PAGER_OK);
1155 }
1156
1157 /*
1158 * A final note: in a low swap situation, we cannot deallocate swap
1159 * and mark a page dirty here because the caller is likely to mark
1160 * the page clean when we return, causing the page to possibly revert
1161 * to all-zero's later.
1162 */
1163}
1164
1165/*
1166 * swap_pager_putpages:
1167 *
1168 * Assign swap (if necessary) and initiate I/O on the specified pages.
1169 *
1170 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects
1171 * are automatically converted to SWAP objects.
1172 *
1173 * In a low memory situation we may block in VOP_STRATEGY(), but the new
1174 * vm_page reservation system coupled with properly written VFS devices
1175 * should ensure that no low-memory deadlock occurs. This is an area
1176 * which needs work.
1177 *
1178 * The parent has N vm_object_pip_add() references prior to
1179 * calling us and will remove references for rtvals[] that are
1180 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1181 * completion.
1182 *
1183 * The parent has soft-busy'd the pages it passes us and will unbusy
1184 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1185 * We need to unbusy the rest on I/O completion.
1186 */
1187void
1188swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1189 boolean_t sync, int *rtvals)
1190{
1191 int i;
1192 int n = 0;
1193
1194 GIANT_REQUIRED;
1195 if (count && m[0]->object != object) {
1196 panic("swap_pager_getpages: object mismatch %p/%p",
1197 object,
1198 m[0]->object
1199 );
1200 }
1201
1202 /*
1203 * Step 1
1204 *
1205 * Turn object into OBJT_SWAP
1206 * check for bogus sysops
1207 * force sync if not pageout process
1208 */
1209 if (object->type != OBJT_SWAP)
1210 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1211 VM_OBJECT_UNLOCK(object);
1212
1213 if (curproc != pageproc)
1214 sync = TRUE;
1215
1216 /*
1217 * Step 2
1218 *
1219 * Update nsw parameters from swap_async_max sysctl values.
1220 * Do not let the sysop crash the machine with bogus numbers.
1221 */
1222 mtx_lock(&pbuf_mtx);
1223 if (swap_async_max != nsw_wcount_async_max) {
1224 int n;
1225 int s;
1226
1227 /*
1228 * limit range
1229 */
1230 if ((n = swap_async_max) > nswbuf / 2)
1231 n = nswbuf / 2;
1232 if (n < 1)
1233 n = 1;
1234 swap_async_max = n;
1235
1236 /*
1237 * Adjust difference ( if possible ). If the current async
1238 * count is too low, we may not be able to make the adjustment
1239 * at this time.
1240 */
1241 s = splvm();
1242 n -= nsw_wcount_async_max;
1243 if (nsw_wcount_async + n >= 0) {
1244 nsw_wcount_async += n;
1245 nsw_wcount_async_max += n;
1246 wakeup(&nsw_wcount_async);
1247 }
1248 splx(s);
1249 }
1250 mtx_unlock(&pbuf_mtx);
1251
1252 /*
1253 * Step 3
1254 *
1255 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1256 * The page is left dirty until the pageout operation completes
1257 * successfully.
1258 */
1259 for (i = 0; i < count; i += n) {
1260 int s;
1261 int j;
1262 struct buf *bp;
1263 daddr_t blk;
1264
1265 /*
1266 * Maximum I/O size is limited by a number of factors.
1267 */
1268 n = min(BLIST_MAX_ALLOC, count - i);
1269 n = min(n, nsw_cluster_max);
1270
1271 s = splvm();
1272
1273 /*
1274 * Get biggest block of swap we can. If we fail, fall
1275 * back and try to allocate a smaller block. Don't go
1276 * overboard trying to allocate space if it would overly
1277 * fragment swap.
1278 */
1279 while (
1280 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1281 n > 4
1282 ) {
1283 n >>= 1;
1284 }
1285 if (blk == SWAPBLK_NONE) {
1286 for (j = 0; j < n; ++j)
1287 rtvals[i+j] = VM_PAGER_FAIL;
1288 splx(s);
1289 continue;
1290 }
1291
1292 /*
1293 * All I/O parameters have been satisfied, build the I/O
1294 * request and assign the swap space.
1295 */
1296 if (sync == TRUE) {
1297 bp = getpbuf(&nsw_wcount_sync);
1298 } else {
1299 bp = getpbuf(&nsw_wcount_async);
1300 bp->b_flags = B_ASYNC;
1301 }
1302 bp->b_flags |= B_PAGING;
1303 bp->b_iocmd = BIO_WRITE;
1304
1305 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1306
1307 bp->b_rcred = crhold(thread0.td_ucred);
1308 bp->b_wcred = crhold(thread0.td_ucred);
1309 bp->b_bcount = PAGE_SIZE * n;
1310 bp->b_bufsize = PAGE_SIZE * n;
1311 bp->b_blkno = blk;
1312
1313 VM_OBJECT_LOCK(object);
1314 for (j = 0; j < n; ++j) {
1315 vm_page_t mreq = m[i+j];
1316
1317 swp_pager_meta_build(
1318 mreq->object,
1319 mreq->pindex,
1320 blk + j
1321 );
1322 vm_page_dirty(mreq);
1323 rtvals[i+j] = VM_PAGER_OK;
1324
1325 vm_page_lock_queues();
1326 vm_page_flag_set(mreq, PG_SWAPINPROG);
1327 vm_page_unlock_queues();
1328 bp->b_pages[j] = mreq;
1329 }
1330 VM_OBJECT_UNLOCK(object);
1331 bp->b_npages = n;
1332 /*
1333 * Must set dirty range for NFS to work.
1334 */
1335 bp->b_dirtyoff = 0;
1336 bp->b_dirtyend = bp->b_bcount;
1337
1338 cnt.v_swapout++;
1339 cnt.v_swappgsout += bp->b_npages;
1340
1341 splx(s);
1342
1343 /*
1344 * asynchronous
1345 *
1346 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1347 */
1348 if (sync == FALSE) {
1349 bp->b_iodone = swp_pager_async_iodone;
1350 BUF_KERNPROC(bp);
1351 swp_pager_strategy(bp);
1352
1353 for (j = 0; j < n; ++j)
1354 rtvals[i+j] = VM_PAGER_PEND;
1355 /* restart outter loop */
1356 continue;
1357 }
1358
1359 /*
1360 * synchronous
1361 *
1362 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1363 */
1364 bp->b_iodone = bdone;
1365 swp_pager_strategy(bp);
1366
1367 /*
1368 * Wait for the sync I/O to complete, then update rtvals.
1369 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1370 * our async completion routine at the end, thus avoiding a
1371 * double-free.
1372 */
1373 s = splbio();
1374 bwait(bp, PVM, "swwrt");
1375 for (j = 0; j < n; ++j)
1376 rtvals[i+j] = VM_PAGER_PEND;
1377 /*
1378 * Now that we are through with the bp, we can call the
1379 * normal async completion, which frees everything up.
1380 */
1381 swp_pager_async_iodone(bp);
1382 splx(s);
1383 }
1384 VM_OBJECT_LOCK(object);
1385}
1386
1387/*
1388 * swp_pager_async_iodone:
1389 *
1390 * Completion routine for asynchronous reads and writes from/to swap.
1391 * Also called manually by synchronous code to finish up a bp.
1392 *
1393 * For READ operations, the pages are PG_BUSY'd. For WRITE operations,
1394 * the pages are vm_page_t->busy'd. For READ operations, we PG_BUSY
1395 * unbusy all pages except the 'main' request page. For WRITE
1396 * operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1397 * because we marked them all VM_PAGER_PEND on return from putpages ).
1398 *
1399 * This routine may not block.
1400 * This routine is called at splbio() or better
1401 *
1402 * We up ourselves to splvm() as required for various vm_page related
1403 * calls.
1404 */
1405static void
1406swp_pager_async_iodone(struct buf *bp)
1407{
1408 int s;
1409 int i;
1410 vm_object_t object = NULL;
1411
1412 bp->b_flags |= B_DONE;
1413
1414 /*
1415 * report error
1416 */
1417 if (bp->b_ioflags & BIO_ERROR) {
1418 printf(
1419 "swap_pager: I/O error - %s failed; blkno %ld,"
1420 "size %ld, error %d\n",
1421 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1422 (long)bp->b_blkno,
1423 (long)bp->b_bcount,
1424 bp->b_error
1425 );
1426 }
1427
1428 /*
1429 * set object, raise to splvm().
1430 */
1431 s = splvm();
1432
1433 /*
1434 * remove the mapping for kernel virtual
1435 */
1436 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1437
1438 if (bp->b_npages) {
1439 object = bp->b_pages[0]->object;
1440 VM_OBJECT_LOCK(object);
1441 }
1442 vm_page_lock_queues();
1443 /*
1444 * cleanup pages. If an error occurs writing to swap, we are in
1445 * very serious trouble. If it happens to be a disk error, though,
1446 * we may be able to recover by reassigning the swap later on. So
1447 * in this case we remove the m->swapblk assignment for the page
1448 * but do not free it in the rlist. The errornous block(s) are thus
1449 * never reallocated as swap. Redirty the page and continue.
1450 */
1451 for (i = 0; i < bp->b_npages; ++i) {
1452 vm_page_t m = bp->b_pages[i];
1453
1454 vm_page_flag_clear(m, PG_SWAPINPROG);
1455
1456 if (bp->b_ioflags & BIO_ERROR) {
1457 /*
1458 * If an error occurs I'd love to throw the swapblk
1459 * away without freeing it back to swapspace, so it
1460 * can never be used again. But I can't from an
1461 * interrupt.
1462 */
1463 if (bp->b_iocmd == BIO_READ) {
1464 /*
1465 * When reading, reqpage needs to stay
1466 * locked for the parent, but all other
1467 * pages can be freed. We still want to
1468 * wakeup the parent waiting on the page,
1469 * though. ( also: pg_reqpage can be -1 and
1470 * not match anything ).
1471 *
1472 * We have to wake specifically requested pages
1473 * up too because we cleared PG_SWAPINPROG and
1474 * someone may be waiting for that.
1475 *
1476 * NOTE: for reads, m->dirty will probably
1477 * be overridden by the original caller of
1478 * getpages so don't play cute tricks here.
1479 *
1480 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1481 * AS THIS MESSES WITH object->memq, and it is
1482 * not legal to mess with object->memq from an
1483 * interrupt.
1484 */
1485 m->valid = 0;
1486 if (i != bp->b_pager.pg_reqpage)
1487 vm_page_free(m);
1488 else
1489 vm_page_flash(m);
1490 /*
1491 * If i == bp->b_pager.pg_reqpage, do not wake
1492 * the page up. The caller needs to.
1493 */
1494 } else {
1495 /*
1496 * If a write error occurs, reactivate page
1497 * so it doesn't clog the inactive list,
1498 * then finish the I/O.
1499 */
1500 vm_page_dirty(m);
1501 vm_page_activate(m);
1502 vm_page_io_finish(m);
1503 }
1504 } else if (bp->b_iocmd == BIO_READ) {
1505 /*
1506 * For read success, clear dirty bits. Nobody should
1507 * have this page mapped but don't take any chances,
1508 * make sure the pmap modify bits are also cleared.
1509 *
1510 * NOTE: for reads, m->dirty will probably be
1511 * overridden by the original caller of getpages so
1512 * we cannot set them in order to free the underlying
1513 * swap in a low-swap situation. I don't think we'd
1514 * want to do that anyway, but it was an optimization
1515 * that existed in the old swapper for a time before
1516 * it got ripped out due to precisely this problem.
1517 *
1518 * If not the requested page then deactivate it.
1519 *
1520 * Note that the requested page, reqpage, is left
1521 * busied, but we still have to wake it up. The
1522 * other pages are released (unbusied) by
1523 * vm_page_wakeup(). We do not set reqpage's
1524 * valid bits here, it is up to the caller.
1525 */
1526 pmap_clear_modify(m);
1527 m->valid = VM_PAGE_BITS_ALL;
1528 vm_page_undirty(m);
1529
1530 /*
1531 * We have to wake specifically requested pages
1532 * up too because we cleared PG_SWAPINPROG and
1533 * could be waiting for it in getpages. However,
1534 * be sure to not unbusy getpages specifically
1535 * requested page - getpages expects it to be
1536 * left busy.
1537 */
1538 if (i != bp->b_pager.pg_reqpage) {
1539 vm_page_deactivate(m);
1540 vm_page_wakeup(m);
1541 } else {
1542 vm_page_flash(m);
1543 }
1544 } else {
1545 /*
1546 * For write success, clear the modify and dirty
1547 * status, then finish the I/O ( which decrements the
1548 * busy count and possibly wakes waiter's up ).
1549 */
1550 pmap_clear_modify(m);
1551 vm_page_undirty(m);
1552 vm_page_io_finish(m);
1553 if (vm_page_count_severe())
1554 vm_page_try_to_cache(m);
1555 }
1556 }
1557 vm_page_unlock_queues();
1558
1559 /*
1560 * adjust pip. NOTE: the original parent may still have its own
1561 * pip refs on the object.
1562 */
1563 if (object != NULL) {
1564 vm_object_pip_wakeupn(object, bp->b_npages);
1565 VM_OBJECT_UNLOCK(object);
1566 }
1567
1568 /*
1569 * release the physical I/O buffer
1570 */
1571 relpbuf(
1572 bp,
1573 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1574 ((bp->b_flags & B_ASYNC) ?
1575 &nsw_wcount_async :
1576 &nsw_wcount_sync
1577 )
1578 )
1579 );
1580 splx(s);
1581}
1582
1583/*
1584 * swap_pager_isswapped:
1585 *
1586 * Return 1 if at least one page in the given object is paged
1587 * out to the given swap device.
1588 *
1589 * This routine may not block.
1590 */
1591int
1592swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1593{
1594 daddr_t index = 0;
1595 int bcount;
1596 int i;
1597
1598 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1599 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1600 struct swblock *swap;
1601
1602 mtx_lock(&swhash_mtx);
1603 if ((swap = *swp_pager_hash(object, index)) != NULL) {
1604 for (i = 0; i < SWAP_META_PAGES; ++i) {
1605 daddr_t v = swap->swb_pages[i];
1606 if (v == SWAPBLK_NONE)
1607 continue;
1608 if (swp_pager_find_dev(v) == sp) {
1609 mtx_unlock(&swhash_mtx);
1610 return 1;
1611 }
1612 }
1613 }
1614 mtx_unlock(&swhash_mtx);
1615 index += SWAP_META_PAGES;
1616 if (index > 0x20000000)
1617 panic("swap_pager_isswapped: failed to locate all swap meta blocks");
1618 }
1619 return 0;
1620}
1621
1622/*
1623 * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1624 *
1625 * This routine dissociates the page at the given index within a
1626 * swap block from its backing store, paging it in if necessary.
1627 * If the page is paged in, it is placed in the inactive queue,
1628 * since it had its backing store ripped out from under it.
1629 * We also attempt to swap in all other pages in the swap block,
1630 * we only guarantee that the one at the specified index is
1631 * paged in.
1632 *
1633 * XXX - The code to page the whole block in doesn't work, so we
1634 * revert to the one-by-one behavior for now. Sigh.
1635 */
1636static __inline void
1637swp_pager_force_pagein(struct swblock *swap, int idx)
1638{
1639 vm_object_t object;
1640 vm_page_t m;
1641 vm_pindex_t pindex;
1642
1643 object = swap->swb_object;
1644 pindex = swap->swb_index;
1645 mtx_unlock(&swhash_mtx);
1646
1647 VM_OBJECT_LOCK(object);
1648 vm_object_pip_add(object, 1);
1649 m = vm_page_grab(object, pindex + idx, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
1650 if (m->valid == VM_PAGE_BITS_ALL) {
1651 vm_object_pip_subtract(object, 1);
1652 vm_page_lock_queues();
1653 vm_page_activate(m);
1654 vm_page_dirty(m);
1655 vm_page_wakeup(m);
1656 vm_page_unlock_queues();
1657 vm_pager_page_unswapped(m);
1658 VM_OBJECT_UNLOCK(object);
1659 return;
1660 }
1661
1662 if (swap_pager_getpages(object, &m, 1, 0) !=
1663 VM_PAGER_OK)
1664 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1665 vm_object_pip_subtract(object, 1);
1666 vm_page_lock_queues();
1667 vm_page_dirty(m);
1668 vm_page_dontneed(m);
1669 vm_page_wakeup(m);
1670 vm_page_unlock_queues();
1671 vm_pager_page_unswapped(m);
1672 VM_OBJECT_UNLOCK(object);
1673}
1674
1675
1676/*
1677 * swap_pager_swapoff:
1678 *
1679 * Page in all of the pages that have been paged out to the
1680 * given device. The corresponding blocks in the bitmap must be
1681 * marked as allocated and the device must be flagged SW_CLOSING.
1682 * There may be no processes swapped out to the device.
1683 *
1684 * The sw_used parameter points to the field in the swdev structure
1685 * that contains a count of the number of blocks still allocated
1686 * on the device. If we encounter objects with a nonzero pip count
1687 * in our scan, we use this number to determine if we're really done.
1688 *
1689 * This routine may block.
1690 */
1691static void
1692swap_pager_swapoff(struct swdevt *sp, int *sw_used)
1693{
1694 struct swblock **pswap;
1695 struct swblock *swap;
1696 vm_object_t waitobj;
1697 daddr_t v;
1698 int i, j;
1699
1700 GIANT_REQUIRED;
1701
1702full_rescan:
1703 waitobj = NULL;
1704 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1705restart:
1706 pswap = &swhash[i];
1707 mtx_lock(&swhash_mtx);
1708 while ((swap = *pswap) != NULL) {
1709 for (j = 0; j < SWAP_META_PAGES; ++j) {
1710 v = swap->swb_pages[j];
1711 if (v != SWAPBLK_NONE &&
1712 swp_pager_find_dev(v) == sp)
1713 break;
1714 }
1715 if (j < SWAP_META_PAGES) {
1716 swp_pager_force_pagein(swap, j);
1717 goto restart;
1718 } else if (swap->swb_object->paging_in_progress) {
1719 if (!waitobj)
1720 waitobj = swap->swb_object;
1721 }
1722 pswap = &swap->swb_hnext;
1723 }
1724 mtx_unlock(&swhash_mtx);
1725 }
1726 if (waitobj && *sw_used) {
1727 /*
1728 * We wait on an arbitrary object to clock our rescans
1729 * to the rate of paging completion.
1730 */
1731 VM_OBJECT_LOCK(waitobj);
1732 vm_object_pip_wait(waitobj, "swpoff");
1733 VM_OBJECT_UNLOCK(waitobj);
1734 goto full_rescan;
1735 }
1736 if (*sw_used)
1737 panic("swapoff: failed to locate %d swap blocks", *sw_used);
1738}
1739
1740/************************************************************************
1741 * SWAP META DATA *
1742 ************************************************************************
1743 *
1744 * These routines manipulate the swap metadata stored in the
1745 * OBJT_SWAP object. All swp_*() routines must be called at
1746 * splvm() because swap can be freed up by the low level vm_page
1747 * code which might be called from interrupts beyond what splbio() covers.
1748 *
1749 * Swap metadata is implemented with a global hash and not directly
1750 * linked into the object. Instead the object simply contains
1751 * appropriate tracking counters.
1752 */
1753
1754/*
1755 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1756 *
1757 * We first convert the object to a swap object if it is a default
1758 * object.
1759 *
1760 * The specified swapblk is added to the object's swap metadata. If
1761 * the swapblk is not valid, it is freed instead. Any previously
1762 * assigned swapblk is freed.
1763 *
1764 * This routine must be called at splvm(), except when used to convert
1765 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1766 */
1767static void
1768swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1769{
1770 struct swblock *swap;
1771 struct swblock **pswap;
1772 int idx;
1773
1774 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1775 /*
1776 * Convert default object to swap object if necessary
1777 */
1778 if (object->type != OBJT_SWAP) {
1779 object->type = OBJT_SWAP;
1780 object->un_pager.swp.swp_bcount = 0;
1781
1782 if (object->handle != NULL) {
1783 mtx_lock(&sw_alloc_mtx);
1784 TAILQ_INSERT_TAIL(
1785 NOBJLIST(object->handle),
1786 object,
1787 pager_object_list
1788 );
1789 mtx_unlock(&sw_alloc_mtx);
1790 }
1791 }
1792
1793 /*
1794 * Locate hash entry. If not found create, but if we aren't adding
1795 * anything just return. If we run out of space in the map we wait
1796 * and, since the hash table may have changed, retry.
1797 */
1798retry:
1799 mtx_lock(&swhash_mtx);
1800 pswap = swp_pager_hash(object, pindex);
1801
1802 if ((swap = *pswap) == NULL) {
1803 int i;
1804
1805 if (swapblk == SWAPBLK_NONE)
1806 goto done;
1807
1808 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
1809 if (swap == NULL) {
1810 mtx_unlock(&swhash_mtx);
1811 VM_OBJECT_UNLOCK(object);
1812 VM_WAIT;
1813 VM_OBJECT_LOCK(object);
1814 goto retry;
1815 }
1816
1817 swap->swb_hnext = NULL;
1818 swap->swb_object = object;
1819 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1820 swap->swb_count = 0;
1821
1822 ++object->un_pager.swp.swp_bcount;
1823
1824 for (i = 0; i < SWAP_META_PAGES; ++i)
1825 swap->swb_pages[i] = SWAPBLK_NONE;
1826 }
1827
1828 /*
1829 * Delete prior contents of metadata
1830 */
1831 idx = pindex & SWAP_META_MASK;
1832
1833 if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1834 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1835 --swap->swb_count;
1836 }
1837
1838 /*
1839 * Enter block into metadata
1840 */
1841 swap->swb_pages[idx] = swapblk;
1842 if (swapblk != SWAPBLK_NONE)
1843 ++swap->swb_count;
1844done:
1845 mtx_unlock(&swhash_mtx);
1846}
1847
1848/*
1849 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1850 *
1851 * The requested range of blocks is freed, with any associated swap
1852 * returned to the swap bitmap.
1853 *
1854 * This routine will free swap metadata structures as they are cleaned
1855 * out. This routine does *NOT* operate on swap metadata associated
1856 * with resident pages.
1857 *
1858 * This routine must be called at splvm()
1859 */
1860static void
1861swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1862{
1863
1864 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1865 if (object->type != OBJT_SWAP)
1866 return;
1867
1868 while (count > 0) {
1869 struct swblock **pswap;
1870 struct swblock *swap;
1871
1872 mtx_lock(&swhash_mtx);
1873 pswap = swp_pager_hash(object, index);
1874
1875 if ((swap = *pswap) != NULL) {
1876 daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1877
1878 if (v != SWAPBLK_NONE) {
1879 swp_pager_freeswapspace(v, 1);
1880 swap->swb_pages[index & SWAP_META_MASK] =
1881 SWAPBLK_NONE;
1882 if (--swap->swb_count == 0) {
1883 *pswap = swap->swb_hnext;
1884 uma_zfree(swap_zone, swap);
1885 --object->un_pager.swp.swp_bcount;
1886 }
1887 }
1888 --count;
1889 ++index;
1890 } else {
1891 int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1892 count -= n;
1893 index += n;
1894 }
1895 mtx_unlock(&swhash_mtx);
1896 }
1897}
1898
1899/*
1900 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1901 *
1902 * This routine locates and destroys all swap metadata associated with
1903 * an object.
1904 *
1905 * This routine must be called at splvm()
1906 */
1907static void
1908swp_pager_meta_free_all(vm_object_t object)
1909{
1910 daddr_t index = 0;
1911
1912 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1913 if (object->type != OBJT_SWAP)
1914 return;
1915
1916 while (object->un_pager.swp.swp_bcount) {
1917 struct swblock **pswap;
1918 struct swblock *swap;
1919
1920 mtx_lock(&swhash_mtx);
1921 pswap = swp_pager_hash(object, index);
1922 if ((swap = *pswap) != NULL) {
1923 int i;
1924
1925 for (i = 0; i < SWAP_META_PAGES; ++i) {
1926 daddr_t v = swap->swb_pages[i];
1927 if (v != SWAPBLK_NONE) {
1928 --swap->swb_count;
1929 swp_pager_freeswapspace(v, 1);
1930 }
1931 }
1932 if (swap->swb_count != 0)
1933 panic("swap_pager_meta_free_all: swb_count != 0");
1934 *pswap = swap->swb_hnext;
1935 uma_zfree(swap_zone, swap);
1936 --object->un_pager.swp.swp_bcount;
1937 }
1938 mtx_unlock(&swhash_mtx);
1939 index += SWAP_META_PAGES;
1940 if (index > 0x20000000)
1941 panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
1942 }
1943}
1944
1945/*
1946 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data.
1947 *
1948 * This routine is capable of looking up, popping, or freeing
1949 * swapblk assignments in the swap meta data or in the vm_page_t.
1950 * The routine typically returns the swapblk being looked-up, or popped,
1951 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1952 * was invalid. This routine will automatically free any invalid
1953 * meta-data swapblks.
1954 *
1955 * It is not possible to store invalid swapblks in the swap meta data
1956 * (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1957 *
1958 * When acting on a busy resident page and paging is in progress, we
1959 * have to wait until paging is complete but otherwise can act on the
1960 * busy page.
1961 *
1962 * This routine must be called at splvm().
1963 *
1964 * SWM_FREE remove and free swap block from metadata
1965 * SWM_POP remove from meta data but do not free.. pop it out
1966 */
1967static daddr_t
1968swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
1969{
1970 struct swblock **pswap;
1971 struct swblock *swap;
1972 daddr_t r1;
1973 int idx;
1974
1975 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1976 /*
1977 * The meta data only exists of the object is OBJT_SWAP
1978 * and even then might not be allocated yet.
1979 */
1980 if (object->type != OBJT_SWAP)
1981 return (SWAPBLK_NONE);
1982
1983 r1 = SWAPBLK_NONE;
1984 mtx_lock(&swhash_mtx);
1985 pswap = swp_pager_hash(object, pindex);
1986
1987 if ((swap = *pswap) != NULL) {
1988 idx = pindex & SWAP_META_MASK;
1989 r1 = swap->swb_pages[idx];
1990
1991 if (r1 != SWAPBLK_NONE) {
1992 if (flags & SWM_FREE) {
1993 swp_pager_freeswapspace(r1, 1);
1994 r1 = SWAPBLK_NONE;
1995 }
1996 if (flags & (SWM_FREE|SWM_POP)) {
1997 swap->swb_pages[idx] = SWAPBLK_NONE;
1998 if (--swap->swb_count == 0) {
1999 *pswap = swap->swb_hnext;
2000 uma_zfree(swap_zone, swap);
2001 --object->un_pager.swp.swp_bcount;
2002 }
2003 }
2004 }
2005 }
2006 mtx_unlock(&swhash_mtx);
2007 return (r1);
2008}
2009
2010/*
2011 * System call swapon(name) enables swapping on device name,
2012 * which must be in the swdevsw. Return EBUSY
2013 * if already swapping on this device.
2014 */
2015#ifndef _SYS_SYSPROTO_H_
2016struct swapon_args {
2017 char *name;
2018};
2019#endif
2020
2021/*
2022 * MPSAFE
2023 */
2024/* ARGSUSED */
2025int
2026swapon(struct thread *td, struct swapon_args *uap)
2027{
2028 struct vattr attr;
2029 struct vnode *vp;
2030 struct nameidata nd;
2031 int error;
2032
2033 mtx_lock(&Giant);
2034 error = suser(td);
2035 if (error)
2036 goto done2;
2037
2038 while (swdev_syscall_active)
2039 tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2040 swdev_syscall_active = 1;
2041
2042 /*
2043 * Swap metadata may not fit in the KVM if we have physical
2044 * memory of >1GB.
2045 */
2046 if (swap_zone == NULL) {
2047 error = ENOMEM;
2048 goto done;
2049 }
2050
2051 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2052 error = namei(&nd);
2053 if (error)
2054 goto done;
2055
2056 NDFREE(&nd, NDF_ONLY_PNBUF);
2057 vp = nd.ni_vp;
2058
2059 if (vn_isdisk(vp, &error)) {
2060 error = swapongeom(td, vp);
2061 } else if (vp->v_type == VREG &&
2062 (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2063 (error = VOP_GETATTR(vp, &attr, td->td_ucred, td)) == 0) {
2064 /*
2065 * Allow direct swapping to NFS regular files in the same
2066 * way that nfs_mountroot() sets up diskless swapping.
2067 */
2068 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2069 }
2070
2071 if (error)
2072 vrele(vp);
2073done:
2074 swdev_syscall_active = 0;
2075 wakeup_one(&swdev_syscall_active);
2076done2:
2077 mtx_unlock(&Giant);
2078 return (error);
2079}
2080
2081static void
| 158 struct vnode *sw_vp;
159 void *sw_id;
160 swblk_t sw_first;
161 swblk_t sw_end;
162 struct blist *sw_blist;
163 TAILQ_ENTRY(swdevt) sw_list;
164 sw_strategy_t *sw_strategy;
165 sw_close_t *sw_close;
166};
167
168#define SW_CLOSING 0x04
169
170struct swblock {
171 struct swblock *swb_hnext;
172 vm_object_t swb_object;
173 vm_pindex_t swb_index;
174 int swb_count;
175 daddr_t swb_pages[SWAP_META_PAGES];
176};
177
178static struct mtx sw_dev_mtx;
179static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
180static struct swdevt *swdevhd; /* Allocate from here next */
181static int nswapdev; /* Number of swap devices */
182int swap_pager_avail;
183static int swdev_syscall_active = 0; /* serialize swap(on|off) */
184
185static void swapdev_strategy(struct buf *, struct swdevt *sw);
186
187#define SWM_FREE 0x02 /* free, period */
188#define SWM_POP 0x04 /* pop out */
189
190int swap_pager_full = 2; /* swap space exhaustion (task killing) */
191static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
192static int nsw_rcount; /* free read buffers */
193static int nsw_wcount_sync; /* limit write buffers / synchronous */
194static int nsw_wcount_async; /* limit write buffers / asynchronous */
195static int nsw_wcount_async_max;/* assigned maximum */
196static int nsw_cluster_max; /* maximum VOP I/O allowed */
197
198static struct swblock **swhash;
199static int swhash_mask;
200static struct mtx swhash_mtx;
201
202static int swap_async_max = 4; /* maximum in-progress async I/O's */
203static struct sx sw_alloc_sx;
204
205
206SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
207 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
208
209/*
210 * "named" and "unnamed" anon region objects. Try to reduce the overhead
211 * of searching a named list by hashing it just a little.
212 */
213
214#define NOBJLISTS 8
215
216#define NOBJLIST(handle) \
217 (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
218
219static struct mtx sw_alloc_mtx; /* protect list manipulation */
220static struct pagerlst swap_pager_object_list[NOBJLISTS];
221static uma_zone_t swap_zone;
222
223/*
224 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure
225 * calls hooked from other parts of the VM system and do not appear here.
226 * (see vm/swap_pager.h).
227 */
228static vm_object_t
229 swap_pager_alloc(void *handle, vm_ooffset_t size,
230 vm_prot_t prot, vm_ooffset_t offset);
231static void swap_pager_dealloc(vm_object_t object);
232static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
233static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
234static boolean_t
235 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
236static void swap_pager_init(void);
237static void swap_pager_unswapped(vm_page_t);
238static void swap_pager_swapoff(struct swdevt *sp, int *sw_used);
239
240struct pagerops swappagerops = {
241 .pgo_init = swap_pager_init, /* early system initialization of pager */
242 .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */
243 .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */
244 .pgo_getpages = swap_pager_getpages, /* pagein */
245 .pgo_putpages = swap_pager_putpages, /* pageout */
246 .pgo_haspage = swap_pager_haspage, /* get backing store status for page */
247 .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */
248};
249
250/*
251 * dmmax is in page-sized chunks with the new swap system. It was
252 * dev-bsized chunks in the old. dmmax is always a power of 2.
253 *
254 * swap_*() routines are externally accessible. swp_*() routines are
255 * internal.
256 */
257static int dmmax;
258static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */
259static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */
260
261SYSCTL_INT(_vm, OID_AUTO, dmmax,
262 CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
263
264static void swp_sizecheck(void);
265static void swp_pager_async_iodone(struct buf *bp);
266static int swapongeom(struct thread *, struct vnode *);
267static int swaponvp(struct thread *, struct vnode *, u_long);
268
269/*
270 * Swap bitmap functions
271 */
272static void swp_pager_freeswapspace(daddr_t blk, int npages);
273static daddr_t swp_pager_getswapspace(int npages);
274
275/*
276 * Metadata functions
277 */
278static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
279static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
280static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
281static void swp_pager_meta_free_all(vm_object_t);
282static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
283
284/*
285 * SWP_SIZECHECK() - update swap_pager_full indication
286 *
287 * update the swap_pager_almost_full indication and warn when we are
288 * about to run out of swap space, using lowat/hiwat hysteresis.
289 *
290 * Clear swap_pager_full ( task killing ) indication when lowat is met.
291 *
292 * No restrictions on call
293 * This routine may not block.
294 * This routine must be called at splvm()
295 */
296static void
297swp_sizecheck(void)
298{
299
300 if (swap_pager_avail < nswap_lowat) {
301 if (swap_pager_almost_full == 0) {
302 printf("swap_pager: out of swap space\n");
303 swap_pager_almost_full = 1;
304 }
305 } else {
306 swap_pager_full = 0;
307 if (swap_pager_avail > nswap_hiwat)
308 swap_pager_almost_full = 0;
309 }
310}
311
312/*
313 * SWP_PAGER_HASH() - hash swap meta data
314 *
315 * This is an helper function which hashes the swapblk given
316 * the object and page index. It returns a pointer to a pointer
317 * to the object, or a pointer to a NULL pointer if it could not
318 * find a swapblk.
319 *
320 * This routine must be called at splvm().
321 */
322static struct swblock **
323swp_pager_hash(vm_object_t object, vm_pindex_t index)
324{
325 struct swblock **pswap;
326 struct swblock *swap;
327
328 index &= ~(vm_pindex_t)SWAP_META_MASK;
329 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
330 while ((swap = *pswap) != NULL) {
331 if (swap->swb_object == object &&
332 swap->swb_index == index
333 ) {
334 break;
335 }
336 pswap = &swap->swb_hnext;
337 }
338 return (pswap);
339}
340
341/*
342 * SWAP_PAGER_INIT() - initialize the swap pager!
343 *
344 * Expected to be started from system init. NOTE: This code is run
345 * before much else so be careful what you depend on. Most of the VM
346 * system has yet to be initialized at this point.
347 */
348static void
349swap_pager_init(void)
350{
351 /*
352 * Initialize object lists
353 */
354 int i;
355
356 for (i = 0; i < NOBJLISTS; ++i)
357 TAILQ_INIT(&swap_pager_object_list[i]);
358 mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
359 mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
360
361 /*
362 * Device Stripe, in PAGE_SIZE'd blocks
363 */
364 dmmax = SWB_NPAGES * 2;
365}
366
367/*
368 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
369 *
370 * Expected to be started from pageout process once, prior to entering
371 * its main loop.
372 */
373void
374swap_pager_swap_init(void)
375{
376 int n, n2;
377
378 /*
379 * Number of in-transit swap bp operations. Don't
380 * exhaust the pbufs completely. Make sure we
381 * initialize workable values (0 will work for hysteresis
382 * but it isn't very efficient).
383 *
384 * The nsw_cluster_max is constrained by the bp->b_pages[]
385 * array (MAXPHYS/PAGE_SIZE) and our locally defined
386 * MAX_PAGEOUT_CLUSTER. Also be aware that swap ops are
387 * constrained by the swap device interleave stripe size.
388 *
389 * Currently we hardwire nsw_wcount_async to 4. This limit is
390 * designed to prevent other I/O from having high latencies due to
391 * our pageout I/O. The value 4 works well for one or two active swap
392 * devices but is probably a little low if you have more. Even so,
393 * a higher value would probably generate only a limited improvement
394 * with three or four active swap devices since the system does not
395 * typically have to pageout at extreme bandwidths. We will want
396 * at least 2 per swap devices, and 4 is a pretty good value if you
397 * have one NFS swap device due to the command/ack latency over NFS.
398 * So it all works out pretty well.
399 */
400 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
401
402 mtx_lock(&pbuf_mtx);
403 nsw_rcount = (nswbuf + 1) / 2;
404 nsw_wcount_sync = (nswbuf + 3) / 4;
405 nsw_wcount_async = 4;
406 nsw_wcount_async_max = nsw_wcount_async;
407 mtx_unlock(&pbuf_mtx);
408
409 /*
410 * Initialize our zone. Right now I'm just guessing on the number
411 * we need based on the number of pages in the system. Each swblock
412 * can hold 16 pages, so this is probably overkill. This reservation
413 * is typically limited to around 32MB by default.
414 */
415 n = cnt.v_page_count / 2;
416 if (maxswzone && n > maxswzone / sizeof(struct swblock))
417 n = maxswzone / sizeof(struct swblock);
418 n2 = n;
419 swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
420 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
421 do {
422 if (uma_zone_set_obj(swap_zone, NULL, n))
423 break;
424 /*
425 * if the allocation failed, try a zone two thirds the
426 * size of the previous attempt.
427 */
428 n -= ((n + 2) / 3);
429 } while (n > 0);
430 if (swap_zone == NULL)
431 panic("failed to create swap_zone.");
432 if (n2 != n)
433 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
434 n2 = n;
435
436 /*
437 * Initialize our meta-data hash table. The swapper does not need to
438 * be quite as efficient as the VM system, so we do not use an
439 * oversized hash table.
440 *
441 * n: size of hash table, must be power of 2
442 * swhash_mask: hash table index mask
443 */
444 for (n = 1; n < n2 / 8; n *= 2)
445 ;
446 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
447 swhash_mask = n - 1;
448 mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
449}
450
451/*
452 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
453 * its metadata structures.
454 *
455 * This routine is called from the mmap and fork code to create a new
456 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
457 * and then converting it with swp_pager_meta_build().
458 *
459 * This routine may block in vm_object_allocate() and create a named
460 * object lookup race, so we must interlock. We must also run at
461 * splvm() for the object lookup to handle races with interrupts, but
462 * we do not have to maintain splvm() in between the lookup and the
463 * add because (I believe) it is not possible to attempt to create
464 * a new swap object w/handle when a default object with that handle
465 * already exists.
466 *
467 * MPSAFE
468 */
469static vm_object_t
470swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
471 vm_ooffset_t offset)
472{
473 vm_object_t object;
474 vm_pindex_t pindex;
475
476 pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
477
478 if (handle) {
479 mtx_lock(&Giant);
480 /*
481 * Reference existing named region or allocate new one. There
482 * should not be a race here against swp_pager_meta_build()
483 * as called from vm_page_remove() in regards to the lookup
484 * of the handle.
485 */
486 sx_xlock(&sw_alloc_sx);
487 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
488
489 if (object != NULL) {
490 vm_object_reference(object);
491 } else {
492 object = vm_object_allocate(OBJT_DEFAULT, pindex);
493 object->handle = handle;
494
495 VM_OBJECT_LOCK(object);
496 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
497 VM_OBJECT_UNLOCK(object);
498 }
499 sx_xunlock(&sw_alloc_sx);
500 mtx_unlock(&Giant);
501 } else {
502 object = vm_object_allocate(OBJT_DEFAULT, pindex);
503
504 VM_OBJECT_LOCK(object);
505 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
506 VM_OBJECT_UNLOCK(object);
507 }
508 return (object);
509}
510
511/*
512 * SWAP_PAGER_DEALLOC() - remove swap metadata from object
513 *
514 * The swap backing for the object is destroyed. The code is
515 * designed such that we can reinstantiate it later, but this
516 * routine is typically called only when the entire object is
517 * about to be destroyed.
518 *
519 * This routine may block, but no longer does.
520 *
521 * The object must be locked or unreferenceable.
522 */
523static void
524swap_pager_dealloc(vm_object_t object)
525{
526 int s;
527
528 /*
529 * Remove from list right away so lookups will fail if we block for
530 * pageout completion.
531 */
532 if (object->handle != NULL) {
533 mtx_lock(&sw_alloc_mtx);
534 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
535 mtx_unlock(&sw_alloc_mtx);
536 }
537
538 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
539 vm_object_pip_wait(object, "swpdea");
540
541 /*
542 * Free all remaining metadata. We only bother to free it from
543 * the swap meta data. We do not attempt to free swapblk's still
544 * associated with vm_page_t's for this object. We do not care
545 * if paging is still in progress on some objects.
546 */
547 s = splvm();
548 swp_pager_meta_free_all(object);
549 splx(s);
550}
551
552/************************************************************************
553 * SWAP PAGER BITMAP ROUTINES *
554 ************************************************************************/
555
556/*
557 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space
558 *
559 * Allocate swap for the requested number of pages. The starting
560 * swap block number (a page index) is returned or SWAPBLK_NONE
561 * if the allocation failed.
562 *
563 * Also has the side effect of advising that somebody made a mistake
564 * when they configured swap and didn't configure enough.
565 *
566 * Must be called at splvm() to avoid races with bitmap frees from
567 * vm_page_remove() aka swap_pager_page_removed().
568 *
569 * This routine may not block
570 * This routine must be called at splvm().
571 *
572 * We allocate in round-robin fashion from the configured devices.
573 */
574static daddr_t
575swp_pager_getswapspace(int npages)
576{
577 daddr_t blk;
578 struct swdevt *sp;
579 int i;
580
581 blk = SWAPBLK_NONE;
582 mtx_lock(&sw_dev_mtx);
583 sp = swdevhd;
584 for (i = 0; i < nswapdev; i++) {
585 if (sp == NULL)
586 sp = TAILQ_FIRST(&swtailq);
587 if (!(sp->sw_flags & SW_CLOSING)) {
588 blk = blist_alloc(sp->sw_blist, npages);
589 if (blk != SWAPBLK_NONE) {
590 blk += sp->sw_first;
591 sp->sw_used += npages;
592 swap_pager_avail -= npages;
593 swp_sizecheck();
594 swdevhd = TAILQ_NEXT(sp, sw_list);
595 goto done;
596 }
597 }
598 sp = TAILQ_NEXT(sp, sw_list);
599 }
600 if (swap_pager_full != 2) {
601 printf("swap_pager_getswapspace(%d): failed\n", npages);
602 swap_pager_full = 2;
603 swap_pager_almost_full = 1;
604 }
605 swdevhd = NULL;
606done:
607 mtx_unlock(&sw_dev_mtx);
608 return (blk);
609}
610
611static struct swdevt *
612swp_pager_find_dev(daddr_t blk)
613{
614 struct swdevt *sp;
615
616 mtx_lock(&sw_dev_mtx);
617 TAILQ_FOREACH(sp, &swtailq, sw_list) {
618 if (blk >= sp->sw_first && blk < sp->sw_end) {
619 mtx_unlock(&sw_dev_mtx);
620 return (sp);
621 }
622 }
623 panic("Swapdev not found");
624}
625
626static void
627swp_pager_strategy(struct buf *bp)
628{
629 struct swdevt *sp;
630
631 mtx_lock(&sw_dev_mtx);
632 TAILQ_FOREACH(sp, &swtailq, sw_list) {
633 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
634 mtx_unlock(&sw_dev_mtx);
635 sp->sw_strategy(bp, sp);
636 return;
637 }
638 }
639 panic("Swapdev not found");
640}
641
642
643/*
644 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
645 *
646 * This routine returns the specified swap blocks back to the bitmap.
647 *
648 * Note: This routine may not block (it could in the old swap code),
649 * and through the use of the new blist routines it does not block.
650 *
651 * We must be called at splvm() to avoid races with bitmap frees from
652 * vm_page_remove() aka swap_pager_page_removed().
653 *
654 * This routine may not block
655 * This routine must be called at splvm().
656 */
657static void
658swp_pager_freeswapspace(daddr_t blk, int npages)
659{
660 struct swdevt *sp;
661
662 mtx_lock(&sw_dev_mtx);
663 TAILQ_FOREACH(sp, &swtailq, sw_list) {
664 if (blk >= sp->sw_first && blk < sp->sw_end) {
665 sp->sw_used -= npages;
666 /*
667 * If we are attempting to stop swapping on
668 * this device, we don't want to mark any
669 * blocks free lest they be reused.
670 */
671 if ((sp->sw_flags & SW_CLOSING) == 0) {
672 blist_free(sp->sw_blist, blk - sp->sw_first,
673 npages);
674 swap_pager_avail += npages;
675 swp_sizecheck();
676 }
677 mtx_unlock(&sw_dev_mtx);
678 return;
679 }
680 }
681 panic("Swapdev not found");
682}
683
684/*
685 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
686 * range within an object.
687 *
688 * This is a globally accessible routine.
689 *
690 * This routine removes swapblk assignments from swap metadata.
691 *
692 * The external callers of this routine typically have already destroyed
693 * or renamed vm_page_t's associated with this range in the object so
694 * we should be ok.
695 *
696 * This routine may be called at any spl. We up our spl to splvm temporarily
697 * in order to perform the metadata removal.
698 */
699void
700swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
701{
702 int s = splvm();
703
704 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
705 swp_pager_meta_free(object, start, size);
706 splx(s);
707}
708
709/*
710 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
711 *
712 * Assigns swap blocks to the specified range within the object. The
713 * swap blocks are not zerod. Any previous swap assignment is destroyed.
714 *
715 * Returns 0 on success, -1 on failure.
716 */
717int
718swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
719{
720 int s;
721 int n = 0;
722 daddr_t blk = SWAPBLK_NONE;
723 vm_pindex_t beg = start; /* save start index */
724
725 s = splvm();
726 VM_OBJECT_LOCK(object);
727 while (size) {
728 if (n == 0) {
729 n = BLIST_MAX_ALLOC;
730 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
731 n >>= 1;
732 if (n == 0) {
733 swp_pager_meta_free(object, beg, start - beg);
734 VM_OBJECT_UNLOCK(object);
735 splx(s);
736 return (-1);
737 }
738 }
739 }
740 swp_pager_meta_build(object, start, blk);
741 --size;
742 ++start;
743 ++blk;
744 --n;
745 }
746 swp_pager_meta_free(object, start, n);
747 VM_OBJECT_UNLOCK(object);
748 splx(s);
749 return (0);
750}
751
752/*
753 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
754 * and destroy the source.
755 *
756 * Copy any valid swapblks from the source to the destination. In
757 * cases where both the source and destination have a valid swapblk,
758 * we keep the destination's.
759 *
760 * This routine is allowed to block. It may block allocating metadata
761 * indirectly through swp_pager_meta_build() or if paging is still in
762 * progress on the source.
763 *
764 * This routine can be called at any spl
765 *
766 * XXX vm_page_collapse() kinda expects us not to block because we
767 * supposedly do not need to allocate memory, but for the moment we
768 * *may* have to get a little memory from the zone allocator, but
769 * it is taken from the interrupt memory. We should be ok.
770 *
771 * The source object contains no vm_page_t's (which is just as well)
772 *
773 * The source object is of type OBJT_SWAP.
774 *
775 * The source and destination objects must be locked or
776 * inaccessible (XXX are they ?)
777 */
778void
779swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
780 vm_pindex_t offset, int destroysource)
781{
782 vm_pindex_t i;
783 int s;
784
785 VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
786 VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
787
788 s = splvm();
789 /*
790 * If destroysource is set, we remove the source object from the
791 * swap_pager internal queue now.
792 */
793 if (destroysource) {
794 if (srcobject->handle != NULL) {
795 mtx_lock(&sw_alloc_mtx);
796 TAILQ_REMOVE(
797 NOBJLIST(srcobject->handle),
798 srcobject,
799 pager_object_list
800 );
801 mtx_unlock(&sw_alloc_mtx);
802 }
803 }
804
805 /*
806 * transfer source to destination.
807 */
808 for (i = 0; i < dstobject->size; ++i) {
809 daddr_t dstaddr;
810
811 /*
812 * Locate (without changing) the swapblk on the destination,
813 * unless it is invalid in which case free it silently, or
814 * if the destination is a resident page, in which case the
815 * source is thrown away.
816 */
817 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
818
819 if (dstaddr == SWAPBLK_NONE) {
820 /*
821 * Destination has no swapblk and is not resident,
822 * copy source.
823 */
824 daddr_t srcaddr;
825
826 srcaddr = swp_pager_meta_ctl(
827 srcobject,
828 i + offset,
829 SWM_POP
830 );
831
832 if (srcaddr != SWAPBLK_NONE) {
833 /*
834 * swp_pager_meta_build() can sleep.
835 */
836 vm_object_pip_add(srcobject, 1);
837 VM_OBJECT_UNLOCK(srcobject);
838 vm_object_pip_add(dstobject, 1);
839 swp_pager_meta_build(dstobject, i, srcaddr);
840 vm_object_pip_wakeup(dstobject);
841 VM_OBJECT_LOCK(srcobject);
842 vm_object_pip_wakeup(srcobject);
843 }
844 } else {
845 /*
846 * Destination has valid swapblk or it is represented
847 * by a resident page. We destroy the sourceblock.
848 */
849
850 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
851 }
852 }
853
854 /*
855 * Free left over swap blocks in source.
856 *
857 * We have to revert the type to OBJT_DEFAULT so we do not accidently
858 * double-remove the object from the swap queues.
859 */
860 if (destroysource) {
861 swp_pager_meta_free_all(srcobject);
862 /*
863 * Reverting the type is not necessary, the caller is going
864 * to destroy srcobject directly, but I'm doing it here
865 * for consistency since we've removed the object from its
866 * queues.
867 */
868 srcobject->type = OBJT_DEFAULT;
869 }
870 splx(s);
871}
872
873/*
874 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for
875 * the requested page.
876 *
877 * We determine whether good backing store exists for the requested
878 * page and return TRUE if it does, FALSE if it doesn't.
879 *
880 * If TRUE, we also try to determine how much valid, contiguous backing
881 * store exists before and after the requested page within a reasonable
882 * distance. We do not try to restrict it to the swap device stripe
883 * (that is handled in getpages/putpages). It probably isn't worth
884 * doing here.
885 */
886static boolean_t
887swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
888{
889 daddr_t blk0;
890 int s;
891
892 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
893 /*
894 * do we have good backing store at the requested index ?
895 */
896 s = splvm();
897 blk0 = swp_pager_meta_ctl(object, pindex, 0);
898
899 if (blk0 == SWAPBLK_NONE) {
900 splx(s);
901 if (before)
902 *before = 0;
903 if (after)
904 *after = 0;
905 return (FALSE);
906 }
907
908 /*
909 * find backwards-looking contiguous good backing store
910 */
911 if (before != NULL) {
912 int i;
913
914 for (i = 1; i < (SWB_NPAGES/2); ++i) {
915 daddr_t blk;
916
917 if (i > pindex)
918 break;
919 blk = swp_pager_meta_ctl(object, pindex - i, 0);
920 if (blk != blk0 - i)
921 break;
922 }
923 *before = (i - 1);
924 }
925
926 /*
927 * find forward-looking contiguous good backing store
928 */
929 if (after != NULL) {
930 int i;
931
932 for (i = 1; i < (SWB_NPAGES/2); ++i) {
933 daddr_t blk;
934
935 blk = swp_pager_meta_ctl(object, pindex + i, 0);
936 if (blk != blk0 + i)
937 break;
938 }
939 *after = (i - 1);
940 }
941 splx(s);
942 return (TRUE);
943}
944
945/*
946 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
947 *
948 * This removes any associated swap backing store, whether valid or
949 * not, from the page.
950 *
951 * This routine is typically called when a page is made dirty, at
952 * which point any associated swap can be freed. MADV_FREE also
953 * calls us in a special-case situation
954 *
955 * NOTE!!! If the page is clean and the swap was valid, the caller
956 * should make the page dirty before calling this routine. This routine
957 * does NOT change the m->dirty status of the page. Also: MADV_FREE
958 * depends on it.
959 *
960 * This routine may not block
961 * This routine must be called at splvm()
962 */
963static void
964swap_pager_unswapped(vm_page_t m)
965{
966
967 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
968 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
969}
970
971/*
972 * SWAP_PAGER_GETPAGES() - bring pages in from swap
973 *
974 * Attempt to retrieve (m, count) pages from backing store, but make
975 * sure we retrieve at least m[reqpage]. We try to load in as large
976 * a chunk surrounding m[reqpage] as is contiguous in swap and which
977 * belongs to the same object.
978 *
979 * The code is designed for asynchronous operation and
980 * immediate-notification of 'reqpage' but tends not to be
981 * used that way. Please do not optimize-out this algorithmic
982 * feature, I intend to improve on it in the future.
983 *
984 * The parent has a single vm_object_pip_add() reference prior to
985 * calling us and we should return with the same.
986 *
987 * The parent has BUSY'd the pages. We should return with 'm'
988 * left busy, but the others adjusted.
989 */
990static int
991swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
992{
993 struct buf *bp;
994 vm_page_t mreq;
995 int s;
996 int i;
997 int j;
998 daddr_t blk;
999
1000 mreq = m[reqpage];
1001
1002 KASSERT(mreq->object == object,
1003 ("swap_pager_getpages: object mismatch %p/%p",
1004 object, mreq->object));
1005
1006 /*
1007 * Calculate range to retrieve. The pages have already been assigned
1008 * their swapblks. We require a *contiguous* range but we know it to
1009 * not span devices. If we do not supply it, bad things
1010 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1011 * loops are set up such that the case(s) are handled implicitly.
1012 *
1013 * The swp_*() calls must be made at splvm(). vm_page_free() does
1014 * not need to be, but it will go a little faster if it is.
1015 */
1016 s = splvm();
1017 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1018
1019 for (i = reqpage - 1; i >= 0; --i) {
1020 daddr_t iblk;
1021
1022 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1023 if (blk != iblk + (reqpage - i))
1024 break;
1025 }
1026 ++i;
1027
1028 for (j = reqpage + 1; j < count; ++j) {
1029 daddr_t jblk;
1030
1031 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1032 if (blk != jblk - (j - reqpage))
1033 break;
1034 }
1035
1036 /*
1037 * free pages outside our collection range. Note: we never free
1038 * mreq, it must remain busy throughout.
1039 */
1040 vm_page_lock_queues();
1041 {
1042 int k;
1043
1044 for (k = 0; k < i; ++k)
1045 vm_page_free(m[k]);
1046 for (k = j; k < count; ++k)
1047 vm_page_free(m[k]);
1048 }
1049 vm_page_unlock_queues();
1050 splx(s);
1051
1052
1053 /*
1054 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1055 * still busy, but the others unbusied.
1056 */
1057 if (blk == SWAPBLK_NONE)
1058 return (VM_PAGER_FAIL);
1059
1060 /*
1061 * Getpbuf() can sleep.
1062 */
1063 VM_OBJECT_UNLOCK(object);
1064 /*
1065 * Get a swap buffer header to perform the IO
1066 */
1067 bp = getpbuf(&nsw_rcount);
1068 bp->b_flags |= B_PAGING;
1069
1070 /*
1071 * map our page(s) into kva for input
1072 */
1073 pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
1074
1075 bp->b_iocmd = BIO_READ;
1076 bp->b_iodone = swp_pager_async_iodone;
1077 bp->b_rcred = crhold(thread0.td_ucred);
1078 bp->b_wcred = crhold(thread0.td_ucred);
1079 bp->b_blkno = blk - (reqpage - i);
1080 bp->b_bcount = PAGE_SIZE * (j - i);
1081 bp->b_bufsize = PAGE_SIZE * (j - i);
1082 bp->b_pager.pg_reqpage = reqpage - i;
1083
1084 VM_OBJECT_LOCK(object);
1085 vm_page_lock_queues();
1086 {
1087 int k;
1088
1089 for (k = i; k < j; ++k) {
1090 bp->b_pages[k - i] = m[k];
1091 vm_page_flag_set(m[k], PG_SWAPINPROG);
1092 }
1093 }
1094 vm_page_unlock_queues();
1095 VM_OBJECT_UNLOCK(object);
1096 bp->b_npages = j - i;
1097
1098 cnt.v_swapin++;
1099 cnt.v_swappgsin += bp->b_npages;
1100
1101 /*
1102 * We still hold the lock on mreq, and our automatic completion routine
1103 * does not remove it.
1104 */
1105 VM_OBJECT_LOCK(mreq->object);
1106 vm_object_pip_add(mreq->object, bp->b_npages);
1107 VM_OBJECT_UNLOCK(mreq->object);
1108
1109 /*
1110 * perform the I/O. NOTE!!! bp cannot be considered valid after
1111 * this point because we automatically release it on completion.
1112 * Instead, we look at the one page we are interested in which we
1113 * still hold a lock on even through the I/O completion.
1114 *
1115 * The other pages in our m[] array are also released on completion,
1116 * so we cannot assume they are valid anymore either.
1117 *
1118 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1119 */
1120 BUF_KERNPROC(bp);
1121 swp_pager_strategy(bp);
1122
1123 /*
1124 * wait for the page we want to complete. PG_SWAPINPROG is always
1125 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1126 * is set in the meta-data.
1127 */
1128 s = splvm();
1129 vm_page_lock_queues();
1130 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1131 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1132 cnt.v_intrans++;
1133 if (msleep(mreq, &vm_page_queue_mtx, PSWP, "swread", hz*20)) {
1134 printf(
1135 "swap_pager: indefinite wait buffer: device:"
1136 " %s, blkno: %ld, size: %ld\n",
1137 devtoname(bp->b_dev), (long)bp->b_blkno,
1138 bp->b_bcount
1139 );
1140 }
1141 }
1142 vm_page_unlock_queues();
1143 splx(s);
1144
1145 VM_OBJECT_LOCK(mreq->object);
1146 /*
1147 * mreq is left busied after completion, but all the other pages
1148 * are freed. If we had an unrecoverable read error the page will
1149 * not be valid.
1150 */
1151 if (mreq->valid != VM_PAGE_BITS_ALL) {
1152 return (VM_PAGER_ERROR);
1153 } else {
1154 return (VM_PAGER_OK);
1155 }
1156
1157 /*
1158 * A final note: in a low swap situation, we cannot deallocate swap
1159 * and mark a page dirty here because the caller is likely to mark
1160 * the page clean when we return, causing the page to possibly revert
1161 * to all-zero's later.
1162 */
1163}
1164
1165/*
1166 * swap_pager_putpages:
1167 *
1168 * Assign swap (if necessary) and initiate I/O on the specified pages.
1169 *
1170 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects
1171 * are automatically converted to SWAP objects.
1172 *
1173 * In a low memory situation we may block in VOP_STRATEGY(), but the new
1174 * vm_page reservation system coupled with properly written VFS devices
1175 * should ensure that no low-memory deadlock occurs. This is an area
1176 * which needs work.
1177 *
1178 * The parent has N vm_object_pip_add() references prior to
1179 * calling us and will remove references for rtvals[] that are
1180 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1181 * completion.
1182 *
1183 * The parent has soft-busy'd the pages it passes us and will unbusy
1184 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1185 * We need to unbusy the rest on I/O completion.
1186 */
1187void
1188swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
1189 boolean_t sync, int *rtvals)
1190{
1191 int i;
1192 int n = 0;
1193
1194 GIANT_REQUIRED;
1195 if (count && m[0]->object != object) {
1196 panic("swap_pager_getpages: object mismatch %p/%p",
1197 object,
1198 m[0]->object
1199 );
1200 }
1201
1202 /*
1203 * Step 1
1204 *
1205 * Turn object into OBJT_SWAP
1206 * check for bogus sysops
1207 * force sync if not pageout process
1208 */
1209 if (object->type != OBJT_SWAP)
1210 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1211 VM_OBJECT_UNLOCK(object);
1212
1213 if (curproc != pageproc)
1214 sync = TRUE;
1215
1216 /*
1217 * Step 2
1218 *
1219 * Update nsw parameters from swap_async_max sysctl values.
1220 * Do not let the sysop crash the machine with bogus numbers.
1221 */
1222 mtx_lock(&pbuf_mtx);
1223 if (swap_async_max != nsw_wcount_async_max) {
1224 int n;
1225 int s;
1226
1227 /*
1228 * limit range
1229 */
1230 if ((n = swap_async_max) > nswbuf / 2)
1231 n = nswbuf / 2;
1232 if (n < 1)
1233 n = 1;
1234 swap_async_max = n;
1235
1236 /*
1237 * Adjust difference ( if possible ). If the current async
1238 * count is too low, we may not be able to make the adjustment
1239 * at this time.
1240 */
1241 s = splvm();
1242 n -= nsw_wcount_async_max;
1243 if (nsw_wcount_async + n >= 0) {
1244 nsw_wcount_async += n;
1245 nsw_wcount_async_max += n;
1246 wakeup(&nsw_wcount_async);
1247 }
1248 splx(s);
1249 }
1250 mtx_unlock(&pbuf_mtx);
1251
1252 /*
1253 * Step 3
1254 *
1255 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1256 * The page is left dirty until the pageout operation completes
1257 * successfully.
1258 */
1259 for (i = 0; i < count; i += n) {
1260 int s;
1261 int j;
1262 struct buf *bp;
1263 daddr_t blk;
1264
1265 /*
1266 * Maximum I/O size is limited by a number of factors.
1267 */
1268 n = min(BLIST_MAX_ALLOC, count - i);
1269 n = min(n, nsw_cluster_max);
1270
1271 s = splvm();
1272
1273 /*
1274 * Get biggest block of swap we can. If we fail, fall
1275 * back and try to allocate a smaller block. Don't go
1276 * overboard trying to allocate space if it would overly
1277 * fragment swap.
1278 */
1279 while (
1280 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1281 n > 4
1282 ) {
1283 n >>= 1;
1284 }
1285 if (blk == SWAPBLK_NONE) {
1286 for (j = 0; j < n; ++j)
1287 rtvals[i+j] = VM_PAGER_FAIL;
1288 splx(s);
1289 continue;
1290 }
1291
1292 /*
1293 * All I/O parameters have been satisfied, build the I/O
1294 * request and assign the swap space.
1295 */
1296 if (sync == TRUE) {
1297 bp = getpbuf(&nsw_wcount_sync);
1298 } else {
1299 bp = getpbuf(&nsw_wcount_async);
1300 bp->b_flags = B_ASYNC;
1301 }
1302 bp->b_flags |= B_PAGING;
1303 bp->b_iocmd = BIO_WRITE;
1304
1305 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1306
1307 bp->b_rcred = crhold(thread0.td_ucred);
1308 bp->b_wcred = crhold(thread0.td_ucred);
1309 bp->b_bcount = PAGE_SIZE * n;
1310 bp->b_bufsize = PAGE_SIZE * n;
1311 bp->b_blkno = blk;
1312
1313 VM_OBJECT_LOCK(object);
1314 for (j = 0; j < n; ++j) {
1315 vm_page_t mreq = m[i+j];
1316
1317 swp_pager_meta_build(
1318 mreq->object,
1319 mreq->pindex,
1320 blk + j
1321 );
1322 vm_page_dirty(mreq);
1323 rtvals[i+j] = VM_PAGER_OK;
1324
1325 vm_page_lock_queues();
1326 vm_page_flag_set(mreq, PG_SWAPINPROG);
1327 vm_page_unlock_queues();
1328 bp->b_pages[j] = mreq;
1329 }
1330 VM_OBJECT_UNLOCK(object);
1331 bp->b_npages = n;
1332 /*
1333 * Must set dirty range for NFS to work.
1334 */
1335 bp->b_dirtyoff = 0;
1336 bp->b_dirtyend = bp->b_bcount;
1337
1338 cnt.v_swapout++;
1339 cnt.v_swappgsout += bp->b_npages;
1340
1341 splx(s);
1342
1343 /*
1344 * asynchronous
1345 *
1346 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1347 */
1348 if (sync == FALSE) {
1349 bp->b_iodone = swp_pager_async_iodone;
1350 BUF_KERNPROC(bp);
1351 swp_pager_strategy(bp);
1352
1353 for (j = 0; j < n; ++j)
1354 rtvals[i+j] = VM_PAGER_PEND;
1355 /* restart outter loop */
1356 continue;
1357 }
1358
1359 /*
1360 * synchronous
1361 *
1362 * NOTE: b_blkno is destroyed by the call to swapdev_strategy
1363 */
1364 bp->b_iodone = bdone;
1365 swp_pager_strategy(bp);
1366
1367 /*
1368 * Wait for the sync I/O to complete, then update rtvals.
1369 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1370 * our async completion routine at the end, thus avoiding a
1371 * double-free.
1372 */
1373 s = splbio();
1374 bwait(bp, PVM, "swwrt");
1375 for (j = 0; j < n; ++j)
1376 rtvals[i+j] = VM_PAGER_PEND;
1377 /*
1378 * Now that we are through with the bp, we can call the
1379 * normal async completion, which frees everything up.
1380 */
1381 swp_pager_async_iodone(bp);
1382 splx(s);
1383 }
1384 VM_OBJECT_LOCK(object);
1385}
1386
1387/*
1388 * swp_pager_async_iodone:
1389 *
1390 * Completion routine for asynchronous reads and writes from/to swap.
1391 * Also called manually by synchronous code to finish up a bp.
1392 *
1393 * For READ operations, the pages are PG_BUSY'd. For WRITE operations,
1394 * the pages are vm_page_t->busy'd. For READ operations, we PG_BUSY
1395 * unbusy all pages except the 'main' request page. For WRITE
1396 * operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1397 * because we marked them all VM_PAGER_PEND on return from putpages ).
1398 *
1399 * This routine may not block.
1400 * This routine is called at splbio() or better
1401 *
1402 * We up ourselves to splvm() as required for various vm_page related
1403 * calls.
1404 */
1405static void
1406swp_pager_async_iodone(struct buf *bp)
1407{
1408 int s;
1409 int i;
1410 vm_object_t object = NULL;
1411
1412 bp->b_flags |= B_DONE;
1413
1414 /*
1415 * report error
1416 */
1417 if (bp->b_ioflags & BIO_ERROR) {
1418 printf(
1419 "swap_pager: I/O error - %s failed; blkno %ld,"
1420 "size %ld, error %d\n",
1421 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1422 (long)bp->b_blkno,
1423 (long)bp->b_bcount,
1424 bp->b_error
1425 );
1426 }
1427
1428 /*
1429 * set object, raise to splvm().
1430 */
1431 s = splvm();
1432
1433 /*
1434 * remove the mapping for kernel virtual
1435 */
1436 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1437
1438 if (bp->b_npages) {
1439 object = bp->b_pages[0]->object;
1440 VM_OBJECT_LOCK(object);
1441 }
1442 vm_page_lock_queues();
1443 /*
1444 * cleanup pages. If an error occurs writing to swap, we are in
1445 * very serious trouble. If it happens to be a disk error, though,
1446 * we may be able to recover by reassigning the swap later on. So
1447 * in this case we remove the m->swapblk assignment for the page
1448 * but do not free it in the rlist. The errornous block(s) are thus
1449 * never reallocated as swap. Redirty the page and continue.
1450 */
1451 for (i = 0; i < bp->b_npages; ++i) {
1452 vm_page_t m = bp->b_pages[i];
1453
1454 vm_page_flag_clear(m, PG_SWAPINPROG);
1455
1456 if (bp->b_ioflags & BIO_ERROR) {
1457 /*
1458 * If an error occurs I'd love to throw the swapblk
1459 * away without freeing it back to swapspace, so it
1460 * can never be used again. But I can't from an
1461 * interrupt.
1462 */
1463 if (bp->b_iocmd == BIO_READ) {
1464 /*
1465 * When reading, reqpage needs to stay
1466 * locked for the parent, but all other
1467 * pages can be freed. We still want to
1468 * wakeup the parent waiting on the page,
1469 * though. ( also: pg_reqpage can be -1 and
1470 * not match anything ).
1471 *
1472 * We have to wake specifically requested pages
1473 * up too because we cleared PG_SWAPINPROG and
1474 * someone may be waiting for that.
1475 *
1476 * NOTE: for reads, m->dirty will probably
1477 * be overridden by the original caller of
1478 * getpages so don't play cute tricks here.
1479 *
1480 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1481 * AS THIS MESSES WITH object->memq, and it is
1482 * not legal to mess with object->memq from an
1483 * interrupt.
1484 */
1485 m->valid = 0;
1486 if (i != bp->b_pager.pg_reqpage)
1487 vm_page_free(m);
1488 else
1489 vm_page_flash(m);
1490 /*
1491 * If i == bp->b_pager.pg_reqpage, do not wake
1492 * the page up. The caller needs to.
1493 */
1494 } else {
1495 /*
1496 * If a write error occurs, reactivate page
1497 * so it doesn't clog the inactive list,
1498 * then finish the I/O.
1499 */
1500 vm_page_dirty(m);
1501 vm_page_activate(m);
1502 vm_page_io_finish(m);
1503 }
1504 } else if (bp->b_iocmd == BIO_READ) {
1505 /*
1506 * For read success, clear dirty bits. Nobody should
1507 * have this page mapped but don't take any chances,
1508 * make sure the pmap modify bits are also cleared.
1509 *
1510 * NOTE: for reads, m->dirty will probably be
1511 * overridden by the original caller of getpages so
1512 * we cannot set them in order to free the underlying
1513 * swap in a low-swap situation. I don't think we'd
1514 * want to do that anyway, but it was an optimization
1515 * that existed in the old swapper for a time before
1516 * it got ripped out due to precisely this problem.
1517 *
1518 * If not the requested page then deactivate it.
1519 *
1520 * Note that the requested page, reqpage, is left
1521 * busied, but we still have to wake it up. The
1522 * other pages are released (unbusied) by
1523 * vm_page_wakeup(). We do not set reqpage's
1524 * valid bits here, it is up to the caller.
1525 */
1526 pmap_clear_modify(m);
1527 m->valid = VM_PAGE_BITS_ALL;
1528 vm_page_undirty(m);
1529
1530 /*
1531 * We have to wake specifically requested pages
1532 * up too because we cleared PG_SWAPINPROG and
1533 * could be waiting for it in getpages. However,
1534 * be sure to not unbusy getpages specifically
1535 * requested page - getpages expects it to be
1536 * left busy.
1537 */
1538 if (i != bp->b_pager.pg_reqpage) {
1539 vm_page_deactivate(m);
1540 vm_page_wakeup(m);
1541 } else {
1542 vm_page_flash(m);
1543 }
1544 } else {
1545 /*
1546 * For write success, clear the modify and dirty
1547 * status, then finish the I/O ( which decrements the
1548 * busy count and possibly wakes waiter's up ).
1549 */
1550 pmap_clear_modify(m);
1551 vm_page_undirty(m);
1552 vm_page_io_finish(m);
1553 if (vm_page_count_severe())
1554 vm_page_try_to_cache(m);
1555 }
1556 }
1557 vm_page_unlock_queues();
1558
1559 /*
1560 * adjust pip. NOTE: the original parent may still have its own
1561 * pip refs on the object.
1562 */
1563 if (object != NULL) {
1564 vm_object_pip_wakeupn(object, bp->b_npages);
1565 VM_OBJECT_UNLOCK(object);
1566 }
1567
1568 /*
1569 * release the physical I/O buffer
1570 */
1571 relpbuf(
1572 bp,
1573 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1574 ((bp->b_flags & B_ASYNC) ?
1575 &nsw_wcount_async :
1576 &nsw_wcount_sync
1577 )
1578 )
1579 );
1580 splx(s);
1581}
1582
1583/*
1584 * swap_pager_isswapped:
1585 *
1586 * Return 1 if at least one page in the given object is paged
1587 * out to the given swap device.
1588 *
1589 * This routine may not block.
1590 */
1591int
1592swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1593{
1594 daddr_t index = 0;
1595 int bcount;
1596 int i;
1597
1598 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1599 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1600 struct swblock *swap;
1601
1602 mtx_lock(&swhash_mtx);
1603 if ((swap = *swp_pager_hash(object, index)) != NULL) {
1604 for (i = 0; i < SWAP_META_PAGES; ++i) {
1605 daddr_t v = swap->swb_pages[i];
1606 if (v == SWAPBLK_NONE)
1607 continue;
1608 if (swp_pager_find_dev(v) == sp) {
1609 mtx_unlock(&swhash_mtx);
1610 return 1;
1611 }
1612 }
1613 }
1614 mtx_unlock(&swhash_mtx);
1615 index += SWAP_META_PAGES;
1616 if (index > 0x20000000)
1617 panic("swap_pager_isswapped: failed to locate all swap meta blocks");
1618 }
1619 return 0;
1620}
1621
1622/*
1623 * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
1624 *
1625 * This routine dissociates the page at the given index within a
1626 * swap block from its backing store, paging it in if necessary.
1627 * If the page is paged in, it is placed in the inactive queue,
1628 * since it had its backing store ripped out from under it.
1629 * We also attempt to swap in all other pages in the swap block,
1630 * we only guarantee that the one at the specified index is
1631 * paged in.
1632 *
1633 * XXX - The code to page the whole block in doesn't work, so we
1634 * revert to the one-by-one behavior for now. Sigh.
1635 */
1636static __inline void
1637swp_pager_force_pagein(struct swblock *swap, int idx)
1638{
1639 vm_object_t object;
1640 vm_page_t m;
1641 vm_pindex_t pindex;
1642
1643 object = swap->swb_object;
1644 pindex = swap->swb_index;
1645 mtx_unlock(&swhash_mtx);
1646
1647 VM_OBJECT_LOCK(object);
1648 vm_object_pip_add(object, 1);
1649 m = vm_page_grab(object, pindex + idx, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
1650 if (m->valid == VM_PAGE_BITS_ALL) {
1651 vm_object_pip_subtract(object, 1);
1652 vm_page_lock_queues();
1653 vm_page_activate(m);
1654 vm_page_dirty(m);
1655 vm_page_wakeup(m);
1656 vm_page_unlock_queues();
1657 vm_pager_page_unswapped(m);
1658 VM_OBJECT_UNLOCK(object);
1659 return;
1660 }
1661
1662 if (swap_pager_getpages(object, &m, 1, 0) !=
1663 VM_PAGER_OK)
1664 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
1665 vm_object_pip_subtract(object, 1);
1666 vm_page_lock_queues();
1667 vm_page_dirty(m);
1668 vm_page_dontneed(m);
1669 vm_page_wakeup(m);
1670 vm_page_unlock_queues();
1671 vm_pager_page_unswapped(m);
1672 VM_OBJECT_UNLOCK(object);
1673}
1674
1675
1676/*
1677 * swap_pager_swapoff:
1678 *
1679 * Page in all of the pages that have been paged out to the
1680 * given device. The corresponding blocks in the bitmap must be
1681 * marked as allocated and the device must be flagged SW_CLOSING.
1682 * There may be no processes swapped out to the device.
1683 *
1684 * The sw_used parameter points to the field in the swdev structure
1685 * that contains a count of the number of blocks still allocated
1686 * on the device. If we encounter objects with a nonzero pip count
1687 * in our scan, we use this number to determine if we're really done.
1688 *
1689 * This routine may block.
1690 */
1691static void
1692swap_pager_swapoff(struct swdevt *sp, int *sw_used)
1693{
1694 struct swblock **pswap;
1695 struct swblock *swap;
1696 vm_object_t waitobj;
1697 daddr_t v;
1698 int i, j;
1699
1700 GIANT_REQUIRED;
1701
1702full_rescan:
1703 waitobj = NULL;
1704 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1705restart:
1706 pswap = &swhash[i];
1707 mtx_lock(&swhash_mtx);
1708 while ((swap = *pswap) != NULL) {
1709 for (j = 0; j < SWAP_META_PAGES; ++j) {
1710 v = swap->swb_pages[j];
1711 if (v != SWAPBLK_NONE &&
1712 swp_pager_find_dev(v) == sp)
1713 break;
1714 }
1715 if (j < SWAP_META_PAGES) {
1716 swp_pager_force_pagein(swap, j);
1717 goto restart;
1718 } else if (swap->swb_object->paging_in_progress) {
1719 if (!waitobj)
1720 waitobj = swap->swb_object;
1721 }
1722 pswap = &swap->swb_hnext;
1723 }
1724 mtx_unlock(&swhash_mtx);
1725 }
1726 if (waitobj && *sw_used) {
1727 /*
1728 * We wait on an arbitrary object to clock our rescans
1729 * to the rate of paging completion.
1730 */
1731 VM_OBJECT_LOCK(waitobj);
1732 vm_object_pip_wait(waitobj, "swpoff");
1733 VM_OBJECT_UNLOCK(waitobj);
1734 goto full_rescan;
1735 }
1736 if (*sw_used)
1737 panic("swapoff: failed to locate %d swap blocks", *sw_used);
1738}
1739
1740/************************************************************************
1741 * SWAP META DATA *
1742 ************************************************************************
1743 *
1744 * These routines manipulate the swap metadata stored in the
1745 * OBJT_SWAP object. All swp_*() routines must be called at
1746 * splvm() because swap can be freed up by the low level vm_page
1747 * code which might be called from interrupts beyond what splbio() covers.
1748 *
1749 * Swap metadata is implemented with a global hash and not directly
1750 * linked into the object. Instead the object simply contains
1751 * appropriate tracking counters.
1752 */
1753
1754/*
1755 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1756 *
1757 * We first convert the object to a swap object if it is a default
1758 * object.
1759 *
1760 * The specified swapblk is added to the object's swap metadata. If
1761 * the swapblk is not valid, it is freed instead. Any previously
1762 * assigned swapblk is freed.
1763 *
1764 * This routine must be called at splvm(), except when used to convert
1765 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1766 */
1767static void
1768swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
1769{
1770 struct swblock *swap;
1771 struct swblock **pswap;
1772 int idx;
1773
1774 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1775 /*
1776 * Convert default object to swap object if necessary
1777 */
1778 if (object->type != OBJT_SWAP) {
1779 object->type = OBJT_SWAP;
1780 object->un_pager.swp.swp_bcount = 0;
1781
1782 if (object->handle != NULL) {
1783 mtx_lock(&sw_alloc_mtx);
1784 TAILQ_INSERT_TAIL(
1785 NOBJLIST(object->handle),
1786 object,
1787 pager_object_list
1788 );
1789 mtx_unlock(&sw_alloc_mtx);
1790 }
1791 }
1792
1793 /*
1794 * Locate hash entry. If not found create, but if we aren't adding
1795 * anything just return. If we run out of space in the map we wait
1796 * and, since the hash table may have changed, retry.
1797 */
1798retry:
1799 mtx_lock(&swhash_mtx);
1800 pswap = swp_pager_hash(object, pindex);
1801
1802 if ((swap = *pswap) == NULL) {
1803 int i;
1804
1805 if (swapblk == SWAPBLK_NONE)
1806 goto done;
1807
1808 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
1809 if (swap == NULL) {
1810 mtx_unlock(&swhash_mtx);
1811 VM_OBJECT_UNLOCK(object);
1812 VM_WAIT;
1813 VM_OBJECT_LOCK(object);
1814 goto retry;
1815 }
1816
1817 swap->swb_hnext = NULL;
1818 swap->swb_object = object;
1819 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
1820 swap->swb_count = 0;
1821
1822 ++object->un_pager.swp.swp_bcount;
1823
1824 for (i = 0; i < SWAP_META_PAGES; ++i)
1825 swap->swb_pages[i] = SWAPBLK_NONE;
1826 }
1827
1828 /*
1829 * Delete prior contents of metadata
1830 */
1831 idx = pindex & SWAP_META_MASK;
1832
1833 if (swap->swb_pages[idx] != SWAPBLK_NONE) {
1834 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
1835 --swap->swb_count;
1836 }
1837
1838 /*
1839 * Enter block into metadata
1840 */
1841 swap->swb_pages[idx] = swapblk;
1842 if (swapblk != SWAPBLK_NONE)
1843 ++swap->swb_count;
1844done:
1845 mtx_unlock(&swhash_mtx);
1846}
1847
1848/*
1849 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1850 *
1851 * The requested range of blocks is freed, with any associated swap
1852 * returned to the swap bitmap.
1853 *
1854 * This routine will free swap metadata structures as they are cleaned
1855 * out. This routine does *NOT* operate on swap metadata associated
1856 * with resident pages.
1857 *
1858 * This routine must be called at splvm()
1859 */
1860static void
1861swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1862{
1863
1864 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1865 if (object->type != OBJT_SWAP)
1866 return;
1867
1868 while (count > 0) {
1869 struct swblock **pswap;
1870 struct swblock *swap;
1871
1872 mtx_lock(&swhash_mtx);
1873 pswap = swp_pager_hash(object, index);
1874
1875 if ((swap = *pswap) != NULL) {
1876 daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1877
1878 if (v != SWAPBLK_NONE) {
1879 swp_pager_freeswapspace(v, 1);
1880 swap->swb_pages[index & SWAP_META_MASK] =
1881 SWAPBLK_NONE;
1882 if (--swap->swb_count == 0) {
1883 *pswap = swap->swb_hnext;
1884 uma_zfree(swap_zone, swap);
1885 --object->un_pager.swp.swp_bcount;
1886 }
1887 }
1888 --count;
1889 ++index;
1890 } else {
1891 int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1892 count -= n;
1893 index += n;
1894 }
1895 mtx_unlock(&swhash_mtx);
1896 }
1897}
1898
1899/*
1900 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1901 *
1902 * This routine locates and destroys all swap metadata associated with
1903 * an object.
1904 *
1905 * This routine must be called at splvm()
1906 */
1907static void
1908swp_pager_meta_free_all(vm_object_t object)
1909{
1910 daddr_t index = 0;
1911
1912 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1913 if (object->type != OBJT_SWAP)
1914 return;
1915
1916 while (object->un_pager.swp.swp_bcount) {
1917 struct swblock **pswap;
1918 struct swblock *swap;
1919
1920 mtx_lock(&swhash_mtx);
1921 pswap = swp_pager_hash(object, index);
1922 if ((swap = *pswap) != NULL) {
1923 int i;
1924
1925 for (i = 0; i < SWAP_META_PAGES; ++i) {
1926 daddr_t v = swap->swb_pages[i];
1927 if (v != SWAPBLK_NONE) {
1928 --swap->swb_count;
1929 swp_pager_freeswapspace(v, 1);
1930 }
1931 }
1932 if (swap->swb_count != 0)
1933 panic("swap_pager_meta_free_all: swb_count != 0");
1934 *pswap = swap->swb_hnext;
1935 uma_zfree(swap_zone, swap);
1936 --object->un_pager.swp.swp_bcount;
1937 }
1938 mtx_unlock(&swhash_mtx);
1939 index += SWAP_META_PAGES;
1940 if (index > 0x20000000)
1941 panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
1942 }
1943}
1944
1945/*
1946 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data.
1947 *
1948 * This routine is capable of looking up, popping, or freeing
1949 * swapblk assignments in the swap meta data or in the vm_page_t.
1950 * The routine typically returns the swapblk being looked-up, or popped,
1951 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1952 * was invalid. This routine will automatically free any invalid
1953 * meta-data swapblks.
1954 *
1955 * It is not possible to store invalid swapblks in the swap meta data
1956 * (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1957 *
1958 * When acting on a busy resident page and paging is in progress, we
1959 * have to wait until paging is complete but otherwise can act on the
1960 * busy page.
1961 *
1962 * This routine must be called at splvm().
1963 *
1964 * SWM_FREE remove and free swap block from metadata
1965 * SWM_POP remove from meta data but do not free.. pop it out
1966 */
1967static daddr_t
1968swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
1969{
1970 struct swblock **pswap;
1971 struct swblock *swap;
1972 daddr_t r1;
1973 int idx;
1974
1975 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1976 /*
1977 * The meta data only exists of the object is OBJT_SWAP
1978 * and even then might not be allocated yet.
1979 */
1980 if (object->type != OBJT_SWAP)
1981 return (SWAPBLK_NONE);
1982
1983 r1 = SWAPBLK_NONE;
1984 mtx_lock(&swhash_mtx);
1985 pswap = swp_pager_hash(object, pindex);
1986
1987 if ((swap = *pswap) != NULL) {
1988 idx = pindex & SWAP_META_MASK;
1989 r1 = swap->swb_pages[idx];
1990
1991 if (r1 != SWAPBLK_NONE) {
1992 if (flags & SWM_FREE) {
1993 swp_pager_freeswapspace(r1, 1);
1994 r1 = SWAPBLK_NONE;
1995 }
1996 if (flags & (SWM_FREE|SWM_POP)) {
1997 swap->swb_pages[idx] = SWAPBLK_NONE;
1998 if (--swap->swb_count == 0) {
1999 *pswap = swap->swb_hnext;
2000 uma_zfree(swap_zone, swap);
2001 --object->un_pager.swp.swp_bcount;
2002 }
2003 }
2004 }
2005 }
2006 mtx_unlock(&swhash_mtx);
2007 return (r1);
2008}
2009
2010/*
2011 * System call swapon(name) enables swapping on device name,
2012 * which must be in the swdevsw. Return EBUSY
2013 * if already swapping on this device.
2014 */
2015#ifndef _SYS_SYSPROTO_H_
2016struct swapon_args {
2017 char *name;
2018};
2019#endif
2020
2021/*
2022 * MPSAFE
2023 */
2024/* ARGSUSED */
2025int
2026swapon(struct thread *td, struct swapon_args *uap)
2027{
2028 struct vattr attr;
2029 struct vnode *vp;
2030 struct nameidata nd;
2031 int error;
2032
2033 mtx_lock(&Giant);
2034 error = suser(td);
2035 if (error)
2036 goto done2;
2037
2038 while (swdev_syscall_active)
2039 tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
2040 swdev_syscall_active = 1;
2041
2042 /*
2043 * Swap metadata may not fit in the KVM if we have physical
2044 * memory of >1GB.
2045 */
2046 if (swap_zone == NULL) {
2047 error = ENOMEM;
2048 goto done;
2049 }
2050
2051 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2052 error = namei(&nd);
2053 if (error)
2054 goto done;
2055
2056 NDFREE(&nd, NDF_ONLY_PNBUF);
2057 vp = nd.ni_vp;
2058
2059 if (vn_isdisk(vp, &error)) {
2060 error = swapongeom(td, vp);
2061 } else if (vp->v_type == VREG &&
2062 (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2063 (error = VOP_GETATTR(vp, &attr, td->td_ucred, td)) == 0) {
2064 /*
2065 * Allow direct swapping to NFS regular files in the same
2066 * way that nfs_mountroot() sets up diskless swapping.
2067 */
2068 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2069 }
2070
2071 if (error)
2072 vrele(vp);
2073done:
2074 swdev_syscall_active = 0;
2075 wakeup_one(&swdev_syscall_active);
2076done2:
2077 mtx_unlock(&Giant);
2078 return (error);
2079}
2080
2081static void
|