332 break;
333 /*
334 * if the allocation failed, try a zone two thirds the
335 * size of the previous attempt.
336 */
337 n -= ((n + 2) / 3);
338 } while (n > 0);
339 if (swap_zone == NULL)
340 panic("failed to zinit swap_zone.");
341 if (n2 != n)
342 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
343 n2 = n;
344
345 /*
346 * Initialize our meta-data hash table. The swapper does not need to
347 * be quite as efficient as the VM system, so we do not use an
348 * oversized hash table.
349 *
350 * n: size of hash table, must be power of 2
351 * swhash_mask: hash table index mask
352 */
353 for (n = 1; n < n2 / 8; n *= 2)
354 ;
355 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
356 swhash_mask = n - 1;
357}
358
359/*
360 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
361 * its metadata structures.
362 *
363 * This routine is called from the mmap and fork code to create a new
364 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
365 * and then converting it with swp_pager_meta_build().
366 *
367 * This routine may block in vm_object_allocate() and create a named
368 * object lookup race, so we must interlock. We must also run at
369 * splvm() for the object lookup to handle races with interrupts, but
370 * we do not have to maintain splvm() in between the lookup and the
371 * add because (I believe) it is not possible to attempt to create
372 * a new swap object w/handle when a default object with that handle
373 * already exists.
374 */
375static vm_object_t
376swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
377 vm_ooffset_t offset)
378{
379 vm_object_t object;
380
381 GIANT_REQUIRED;
382
383 if (handle) {
384 /*
385 * Reference existing named region or allocate new one. There
386 * should not be a race here against swp_pager_meta_build()
387 * as called from vm_page_remove() in regards to the lookup
388 * of the handle.
389 */
390 sx_xlock(&sw_alloc_sx);
391 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
392
393 if (object != NULL) {
394 vm_object_reference(object);
395 } else {
396 object = vm_object_allocate(OBJT_DEFAULT,
397 OFF_TO_IDX(offset + PAGE_MASK + size));
398 object->handle = handle;
399
400 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
401 }
402 sx_xunlock(&sw_alloc_sx);
403 } else {
404 object = vm_object_allocate(OBJT_DEFAULT,
405 OFF_TO_IDX(offset + PAGE_MASK + size));
406
407 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
408 }
409
410 return (object);
411}
412
413/*
414 * SWAP_PAGER_DEALLOC() - remove swap metadata from object
415 *
416 * The swap backing for the object is destroyed. The code is
417 * designed such that we can reinstantiate it later, but this
418 * routine is typically called only when the entire object is
419 * about to be destroyed.
420 *
421 * This routine may block, but no longer does.
422 *
423 * The object must be locked or unreferenceable.
424 */
425static void
426swap_pager_dealloc(object)
427 vm_object_t object;
428{
429 int s;
430
431 GIANT_REQUIRED;
432
433 /*
434 * Remove from list right away so lookups will fail if we block for
435 * pageout completion.
436 */
437 mtx_lock(&sw_alloc_mtx);
438 if (object->handle == NULL) {
439 TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list);
440 } else {
441 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
442 }
443 mtx_unlock(&sw_alloc_mtx);
444
445 vm_object_pip_wait(object, "swpdea");
446
447 /*
448 * Free all remaining metadata. We only bother to free it from
449 * the swap meta data. We do not attempt to free swapblk's still
450 * associated with vm_page_t's for this object. We do not care
451 * if paging is still in progress on some objects.
452 */
453 s = splvm();
454 swp_pager_meta_free_all(object);
455 splx(s);
456}
457
458/************************************************************************
459 * SWAP PAGER BITMAP ROUTINES *
460 ************************************************************************/
461
462/*
463 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space
464 *
465 * Allocate swap for the requested number of pages. The starting
466 * swap block number (a page index) is returned or SWAPBLK_NONE
467 * if the allocation failed.
468 *
469 * Also has the side effect of advising that somebody made a mistake
470 * when they configured swap and didn't configure enough.
471 *
472 * Must be called at splvm() to avoid races with bitmap frees from
473 * vm_page_remove() aka swap_pager_page_removed().
474 *
475 * This routine may not block
476 * This routine must be called at splvm().
477 */
478static __inline daddr_t
479swp_pager_getswapspace(npages)
480 int npages;
481{
482 daddr_t blk;
483
484 GIANT_REQUIRED;
485
486 if ((blk = blist_alloc(swapblist, npages)) == SWAPBLK_NONE) {
487 if (swap_pager_full != 2) {
488 printf("swap_pager_getswapspace: failed\n");
489 swap_pager_full = 2;
490 swap_pager_almost_full = 1;
491 }
492 } else {
493 vm_swap_size -= npages;
494 /* per-swap area stats */
495 swdevt[BLK2DEVIDX(blk)].sw_used += npages;
496 swp_sizecheck();
497 }
498 return (blk);
499}
500
501/*
502 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
503 *
504 * This routine returns the specified swap blocks back to the bitmap.
505 *
506 * Note: This routine may not block (it could in the old swap code),
507 * and through the use of the new blist routines it does not block.
508 *
509 * We must be called at splvm() to avoid races with bitmap frees from
510 * vm_page_remove() aka swap_pager_page_removed().
511 *
512 * This routine may not block
513 * This routine must be called at splvm().
514 */
515static __inline void
516swp_pager_freeswapspace(blk, npages)
517 daddr_t blk;
518 int npages;
519{
520 GIANT_REQUIRED;
521
522 blist_free(swapblist, blk, npages);
523 vm_swap_size += npages;
524 /* per-swap area stats */
525 swdevt[BLK2DEVIDX(blk)].sw_used -= npages;
526 swp_sizecheck();
527}
528
529/*
530 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
531 * range within an object.
532 *
533 * This is a globally accessible routine.
534 *
535 * This routine removes swapblk assignments from swap metadata.
536 *
537 * The external callers of this routine typically have already destroyed
538 * or renamed vm_page_t's associated with this range in the object so
539 * we should be ok.
540 *
541 * This routine may be called at any spl. We up our spl to splvm temporarily
542 * in order to perform the metadata removal.
543 */
544void
545swap_pager_freespace(object, start, size)
546 vm_object_t object;
547 vm_pindex_t start;
548 vm_size_t size;
549{
550 int s = splvm();
551
552 GIANT_REQUIRED;
553 swp_pager_meta_free(object, start, size);
554 splx(s);
555}
556
557/*
558 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
559 *
560 * Assigns swap blocks to the specified range within the object. The
561 * swap blocks are not zerod. Any previous swap assignment is destroyed.
562 *
563 * Returns 0 on success, -1 on failure.
564 */
565int
566swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
567{
568 int s;
569 int n = 0;
570 daddr_t blk = SWAPBLK_NONE;
571 vm_pindex_t beg = start; /* save start index */
572
573 s = splvm();
574 while (size) {
575 if (n == 0) {
576 n = BLIST_MAX_ALLOC;
577 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
578 n >>= 1;
579 if (n == 0) {
580 swp_pager_meta_free(object, beg, start - beg);
581 splx(s);
582 return (-1);
583 }
584 }
585 }
586 swp_pager_meta_build(object, start, blk);
587 --size;
588 ++start;
589 ++blk;
590 --n;
591 }
592 swp_pager_meta_free(object, start, n);
593 splx(s);
594 return (0);
595}
596
597/*
598 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
599 * and destroy the source.
600 *
601 * Copy any valid swapblks from the source to the destination. In
602 * cases where both the source and destination have a valid swapblk,
603 * we keep the destination's.
604 *
605 * This routine is allowed to block. It may block allocating metadata
606 * indirectly through swp_pager_meta_build() or if paging is still in
607 * progress on the source.
608 *
609 * This routine can be called at any spl
610 *
611 * XXX vm_page_collapse() kinda expects us not to block because we
612 * supposedly do not need to allocate memory, but for the moment we
613 * *may* have to get a little memory from the zone allocator, but
614 * it is taken from the interrupt memory. We should be ok.
615 *
616 * The source object contains no vm_page_t's (which is just as well)
617 *
618 * The source object is of type OBJT_SWAP.
619 *
620 * The source and destination objects must be locked or
621 * inaccessible (XXX are they ?)
622 */
623void
624swap_pager_copy(srcobject, dstobject, offset, destroysource)
625 vm_object_t srcobject;
626 vm_object_t dstobject;
627 vm_pindex_t offset;
628 int destroysource;
629{
630 vm_pindex_t i;
631 int s;
632
633 GIANT_REQUIRED;
634
635 s = splvm();
636 /*
637 * If destroysource is set, we remove the source object from the
638 * swap_pager internal queue now.
639 */
640 if (destroysource) {
641 mtx_lock(&sw_alloc_mtx);
642 if (srcobject->handle == NULL) {
643 TAILQ_REMOVE(
644 &swap_pager_un_object_list,
645 srcobject,
646 pager_object_list
647 );
648 } else {
649 TAILQ_REMOVE(
650 NOBJLIST(srcobject->handle),
651 srcobject,
652 pager_object_list
653 );
654 }
655 mtx_unlock(&sw_alloc_mtx);
656 }
657
658 /*
659 * transfer source to destination.
660 */
661 for (i = 0; i < dstobject->size; ++i) {
662 daddr_t dstaddr;
663
664 /*
665 * Locate (without changing) the swapblk on the destination,
666 * unless it is invalid in which case free it silently, or
667 * if the destination is a resident page, in which case the
668 * source is thrown away.
669 */
670 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
671
672 if (dstaddr == SWAPBLK_NONE) {
673 /*
674 * Destination has no swapblk and is not resident,
675 * copy source.
676 */
677 daddr_t srcaddr;
678
679 srcaddr = swp_pager_meta_ctl(
680 srcobject,
681 i + offset,
682 SWM_POP
683 );
684
685 if (srcaddr != SWAPBLK_NONE)
686 swp_pager_meta_build(dstobject, i, srcaddr);
687 } else {
688 /*
689 * Destination has valid swapblk or it is represented
690 * by a resident page. We destroy the sourceblock.
691 */
692
693 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
694 }
695 }
696
697 /*
698 * Free left over swap blocks in source.
699 *
700 * We have to revert the type to OBJT_DEFAULT so we do not accidently
701 * double-remove the object from the swap queues.
702 */
703 if (destroysource) {
704 swp_pager_meta_free_all(srcobject);
705 /*
706 * Reverting the type is not necessary, the caller is going
707 * to destroy srcobject directly, but I'm doing it here
708 * for consistency since we've removed the object from its
709 * queues.
710 */
711 srcobject->type = OBJT_DEFAULT;
712 }
713 splx(s);
714}
715
716/*
717 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for
718 * the requested page.
719 *
720 * We determine whether good backing store exists for the requested
721 * page and return TRUE if it does, FALSE if it doesn't.
722 *
723 * If TRUE, we also try to determine how much valid, contiguous backing
724 * store exists before and after the requested page within a reasonable
725 * distance. We do not try to restrict it to the swap device stripe
726 * (that is handled in getpages/putpages). It probably isn't worth
727 * doing here.
728 */
729boolean_t
730swap_pager_haspage(object, pindex, before, after)
731 vm_object_t object;
732 vm_pindex_t pindex;
733 int *before;
734 int *after;
735{
736 daddr_t blk0;
737 int s;
738
739 /*
740 * do we have good backing store at the requested index ?
741 */
742 s = splvm();
743 blk0 = swp_pager_meta_ctl(object, pindex, 0);
744
745 if (blk0 == SWAPBLK_NONE) {
746 splx(s);
747 if (before)
748 *before = 0;
749 if (after)
750 *after = 0;
751 return (FALSE);
752 }
753
754 /*
755 * find backwards-looking contiguous good backing store
756 */
757 if (before != NULL) {
758 int i;
759
760 for (i = 1; i < (SWB_NPAGES/2); ++i) {
761 daddr_t blk;
762
763 if (i > pindex)
764 break;
765 blk = swp_pager_meta_ctl(object, pindex - i, 0);
766 if (blk != blk0 - i)
767 break;
768 }
769 *before = (i - 1);
770 }
771
772 /*
773 * find forward-looking contiguous good backing store
774 */
775 if (after != NULL) {
776 int i;
777
778 for (i = 1; i < (SWB_NPAGES/2); ++i) {
779 daddr_t blk;
780
781 blk = swp_pager_meta_ctl(object, pindex + i, 0);
782 if (blk != blk0 + i)
783 break;
784 }
785 *after = (i - 1);
786 }
787 splx(s);
788 return (TRUE);
789}
790
791/*
792 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
793 *
794 * This removes any associated swap backing store, whether valid or
795 * not, from the page.
796 *
797 * This routine is typically called when a page is made dirty, at
798 * which point any associated swap can be freed. MADV_FREE also
799 * calls us in a special-case situation
800 *
801 * NOTE!!! If the page is clean and the swap was valid, the caller
802 * should make the page dirty before calling this routine. This routine
803 * does NOT change the m->dirty status of the page. Also: MADV_FREE
804 * depends on it.
805 *
806 * This routine may not block
807 * This routine must be called at splvm()
808 */
809static void
810swap_pager_unswapped(m)
811 vm_page_t m;
812{
813 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
814}
815
816/*
817 * SWAP_PAGER_STRATEGY() - read, write, free blocks
818 *
819 * This implements the vm_pager_strategy() interface to swap and allows
820 * other parts of the system to directly access swap as backing store
821 * through vm_objects of type OBJT_SWAP. This is intended to be a
822 * cacheless interface ( i.e. caching occurs at higher levels ).
823 * Therefore we do not maintain any resident pages. All I/O goes
824 * directly to and from the swap device.
825 *
826 * Note that b_blkno is scaled for PAGE_SIZE
827 *
828 * We currently attempt to run I/O synchronously or asynchronously as
829 * the caller requests. This isn't perfect because we loose error
830 * sequencing when we run multiple ops in parallel to satisfy a request.
831 * But this is swap, so we let it all hang out.
832 */
833static void
834swap_pager_strategy(vm_object_t object, struct bio *bp)
835{
836 vm_pindex_t start;
837 int count;
838 int s;
839 char *data;
840 struct buf *nbp = NULL;
841
842 GIANT_REQUIRED;
843
844 /* XXX: KASSERT instead ? */
845 if (bp->bio_bcount & PAGE_MASK) {
846 biofinish(bp, NULL, EINVAL);
847 printf("swap_pager_strategy: bp %p blk %d size %d, not page bounded\n", bp, (int)bp->bio_pblkno, (int)bp->bio_bcount);
848 return;
849 }
850
851 /*
852 * Clear error indication, initialize page index, count, data pointer.
853 */
854 bp->bio_error = 0;
855 bp->bio_flags &= ~BIO_ERROR;
856 bp->bio_resid = bp->bio_bcount;
857 *(u_int *) &bp->bio_driver1 = 0;
858
859 start = bp->bio_pblkno;
860 count = howmany(bp->bio_bcount, PAGE_SIZE);
861 data = bp->bio_data;
862
863 s = splvm();
864
865 /*
866 * Deal with BIO_DELETE
867 */
868 if (bp->bio_cmd == BIO_DELETE) {
869 /*
870 * FREE PAGE(s) - destroy underlying swap that is no longer
871 * needed.
872 */
873 swp_pager_meta_free(object, start, count);
874 splx(s);
875 bp->bio_resid = 0;
876 biodone(bp);
877 return;
878 }
879
880 /*
881 * Execute read or write
882 */
883 while (count > 0) {
884 daddr_t blk;
885
886 /*
887 * Obtain block. If block not found and writing, allocate a
888 * new block and build it into the object.
889 */
890
891 blk = swp_pager_meta_ctl(object, start, 0);
892 if ((blk == SWAPBLK_NONE) && (bp->bio_cmd == BIO_WRITE)) {
893 blk = swp_pager_getswapspace(1);
894 if (blk == SWAPBLK_NONE) {
895 bp->bio_error = ENOMEM;
896 bp->bio_flags |= BIO_ERROR;
897 break;
898 }
899 swp_pager_meta_build(object, start, blk);
900 }
901
902 /*
903 * Do we have to flush our current collection? Yes if:
904 *
905 * - no swap block at this index
906 * - swap block is not contiguous
907 * - we cross a physical disk boundry in the
908 * stripe.
909 */
910 if (
911 nbp && (nbp->b_blkno + btoc(nbp->b_bcount) != blk ||
912 ((nbp->b_blkno ^ blk) & dmmax_mask)
913 )
914 ) {
915 splx(s);
916 if (bp->bio_cmd == BIO_READ) {
917 ++cnt.v_swapin;
918 cnt.v_swappgsin += btoc(nbp->b_bcount);
919 } else {
920 ++cnt.v_swapout;
921 cnt.v_swappgsout += btoc(nbp->b_bcount);
922 nbp->b_dirtyend = nbp->b_bcount;
923 }
924 flushchainbuf(nbp);
925 s = splvm();
926 nbp = NULL;
927 }
928
929 /*
930 * Add new swapblk to nbp, instantiating nbp if necessary.
931 * Zero-fill reads are able to take a shortcut.
932 */
933 if (blk == SWAPBLK_NONE) {
934 /*
935 * We can only get here if we are reading. Since
936 * we are at splvm() we can safely modify b_resid,
937 * even if chain ops are in progress.
938 */
939 bzero(data, PAGE_SIZE);
940 bp->bio_resid -= PAGE_SIZE;
941 } else {
942 if (nbp == NULL) {
943 nbp = getchainbuf(bp, swapdev_vp, B_ASYNC);
944 nbp->b_blkno = blk;
945 nbp->b_bcount = 0;
946 nbp->b_data = data;
947 }
948 nbp->b_bcount += PAGE_SIZE;
949 }
950 --count;
951 ++start;
952 data += PAGE_SIZE;
953 }
954
955 /*
956 * Flush out last buffer
957 */
958 splx(s);
959
960 if (nbp) {
961 if (nbp->b_iocmd == BIO_READ) {
962 ++cnt.v_swapin;
963 cnt.v_swappgsin += btoc(nbp->b_bcount);
964 } else {
965 ++cnt.v_swapout;
966 cnt.v_swappgsout += btoc(nbp->b_bcount);
967 nbp->b_dirtyend = nbp->b_bcount;
968 }
969 flushchainbuf(nbp);
970 /* nbp = NULL; */
971 }
972 /*
973 * Wait for completion.
974 */
975 waitchainbuf(bp, 0, 1);
976}
977
978/*
979 * SWAP_PAGER_GETPAGES() - bring pages in from swap
980 *
981 * Attempt to retrieve (m, count) pages from backing store, but make
982 * sure we retrieve at least m[reqpage]. We try to load in as large
983 * a chunk surrounding m[reqpage] as is contiguous in swap and which
984 * belongs to the same object.
985 *
986 * The code is designed for asynchronous operation and
987 * immediate-notification of 'reqpage' but tends not to be
988 * used that way. Please do not optimize-out this algorithmic
989 * feature, I intend to improve on it in the future.
990 *
991 * The parent has a single vm_object_pip_add() reference prior to
992 * calling us and we should return with the same.
993 *
994 * The parent has BUSY'd the pages. We should return with 'm'
995 * left busy, but the others adjusted.
996 */
997static int
998swap_pager_getpages(object, m, count, reqpage)
999 vm_object_t object;
1000 vm_page_t *m;
1001 int count, reqpage;
1002{
1003 struct buf *bp;
1004 vm_page_t mreq;
1005 int s;
1006 int i;
1007 int j;
1008 daddr_t blk;
1009 vm_offset_t kva;
1010 vm_pindex_t lastpindex;
1011
1012 GIANT_REQUIRED;
1013
1014 mreq = m[reqpage];
1015
1016 if (mreq->object != object) {
1017 panic("swap_pager_getpages: object mismatch %p/%p",
1018 object,
1019 mreq->object
1020 );
1021 }
1022 /*
1023 * Calculate range to retrieve. The pages have already been assigned
1024 * their swapblks. We require a *contiguous* range that falls entirely
1025 * within a single device stripe. If we do not supply it, bad things
1026 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1027 * loops are set up such that the case(s) are handled implicitly.
1028 *
1029 * The swp_*() calls must be made at splvm(). vm_page_free() does
1030 * not need to be, but it will go a little faster if it is.
1031 */
1032 s = splvm();
1033 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1034
1035 for (i = reqpage - 1; i >= 0; --i) {
1036 daddr_t iblk;
1037
1038 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1039 if (blk != iblk + (reqpage - i))
1040 break;
1041 if ((blk ^ iblk) & dmmax_mask)
1042 break;
1043 }
1044 ++i;
1045
1046 for (j = reqpage + 1; j < count; ++j) {
1047 daddr_t jblk;
1048
1049 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1050 if (blk != jblk - (j - reqpage))
1051 break;
1052 if ((blk ^ jblk) & dmmax_mask)
1053 break;
1054 }
1055
1056 /*
1057 * free pages outside our collection range. Note: we never free
1058 * mreq, it must remain busy throughout.
1059 */
1060 {
1061 int k;
1062
1063 for (k = 0; k < i; ++k)
1064 vm_page_free(m[k]);
1065 for (k = j; k < count; ++k)
1066 vm_page_free(m[k]);
1067 }
1068 splx(s);
1069
1070
1071 /*
1072 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1073 * still busy, but the others unbusied.
1074 */
1075 if (blk == SWAPBLK_NONE)
1076 return (VM_PAGER_FAIL);
1077
1078 /*
1079 * Get a swap buffer header to perform the IO
1080 */
1081 bp = getpbuf(&nsw_rcount);
1082 kva = (vm_offset_t) bp->b_data;
1083
1084 /*
1085 * map our page(s) into kva for input
1086 *
1087 * NOTE: B_PAGING is set by pbgetvp()
1088 */
1089 pmap_qenter(kva, m + i, j - i);
1090
1091 bp->b_iocmd = BIO_READ;
1092 bp->b_iodone = swp_pager_async_iodone;
1093 bp->b_rcred = crhold(thread0.td_ucred);
1094 bp->b_wcred = crhold(thread0.td_ucred);
1095 bp->b_data = (caddr_t) kva;
1096 bp->b_blkno = blk - (reqpage - i);
1097 bp->b_bcount = PAGE_SIZE * (j - i);
1098 bp->b_bufsize = PAGE_SIZE * (j - i);
1099 bp->b_pager.pg_reqpage = reqpage - i;
1100
1101 {
1102 int k;
1103
1104 for (k = i; k < j; ++k) {
1105 bp->b_pages[k - i] = m[k];
1106 vm_page_flag_set(m[k], PG_SWAPINPROG);
1107 }
1108 }
1109 bp->b_npages = j - i;
1110
1111 pbgetvp(swapdev_vp, bp);
1112
1113 cnt.v_swapin++;
1114 cnt.v_swappgsin += bp->b_npages;
1115
1116 /*
1117 * We still hold the lock on mreq, and our automatic completion routine
1118 * does not remove it.
1119 */
1120 vm_object_pip_add(mreq->object, bp->b_npages);
1121 lastpindex = m[j-1]->pindex;
1122
1123 /*
1124 * perform the I/O. NOTE!!! bp cannot be considered valid after
1125 * this point because we automatically release it on completion.
1126 * Instead, we look at the one page we are interested in which we
1127 * still hold a lock on even through the I/O completion.
1128 *
1129 * The other pages in our m[] array are also released on completion,
1130 * so we cannot assume they are valid anymore either.
1131 *
1132 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1133 */
1134 BUF_KERNPROC(bp);
1135 BUF_STRATEGY(bp);
1136
1137 /*
1138 * wait for the page we want to complete. PG_SWAPINPROG is always
1139 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1140 * is set in the meta-data.
1141 */
1142 s = splvm();
1143 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1144 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1145 cnt.v_intrans++;
1146 if (tsleep(mreq, PSWP, "swread", hz*20)) {
1147 printf(
1148 "swap_pager: indefinite wait buffer: device:"
1149 " %s, blkno: %ld, size: %ld\n",
1150 devtoname(bp->b_dev), (long)bp->b_blkno,
1151 bp->b_bcount
1152 );
1153 }
1154 }
1155 splx(s);
1156
1157 /*
1158 * mreq is left busied after completion, but all the other pages
1159 * are freed. If we had an unrecoverable read error the page will
1160 * not be valid.
1161 */
1162 if (mreq->valid != VM_PAGE_BITS_ALL) {
1163 return (VM_PAGER_ERROR);
1164 } else {
1165 return (VM_PAGER_OK);
1166 }
1167
1168 /*
1169 * A final note: in a low swap situation, we cannot deallocate swap
1170 * and mark a page dirty here because the caller is likely to mark
1171 * the page clean when we return, causing the page to possibly revert
1172 * to all-zero's later.
1173 */
1174}
1175
1176/*
1177 * swap_pager_putpages:
1178 *
1179 * Assign swap (if necessary) and initiate I/O on the specified pages.
1180 *
1181 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects
1182 * are automatically converted to SWAP objects.
1183 *
1184 * In a low memory situation we may block in VOP_STRATEGY(), but the new
1185 * vm_page reservation system coupled with properly written VFS devices
1186 * should ensure that no low-memory deadlock occurs. This is an area
1187 * which needs work.
1188 *
1189 * The parent has N vm_object_pip_add() references prior to
1190 * calling us and will remove references for rtvals[] that are
1191 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1192 * completion.
1193 *
1194 * The parent has soft-busy'd the pages it passes us and will unbusy
1195 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1196 * We need to unbusy the rest on I/O completion.
1197 */
1198void
1199swap_pager_putpages(object, m, count, sync, rtvals)
1200 vm_object_t object;
1201 vm_page_t *m;
1202 int count;
1203 boolean_t sync;
1204 int *rtvals;
1205{
1206 int i;
1207 int n = 0;
1208
1209 GIANT_REQUIRED;
1210 if (count && m[0]->object != object) {
1211 panic("swap_pager_getpages: object mismatch %p/%p",
1212 object,
1213 m[0]->object
1214 );
1215 }
1216 /*
1217 * Step 1
1218 *
1219 * Turn object into OBJT_SWAP
1220 * check for bogus sysops
1221 * force sync if not pageout process
1222 */
1223 if (object->type != OBJT_SWAP)
1224 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1225
1226 if (curproc != pageproc)
1227 sync = TRUE;
1228
1229 /*
1230 * Step 2
1231 *
1232 * Update nsw parameters from swap_async_max sysctl values.
1233 * Do not let the sysop crash the machine with bogus numbers.
1234 */
1235 mtx_lock(&pbuf_mtx);
1236 if (swap_async_max != nsw_wcount_async_max) {
1237 int n;
1238 int s;
1239
1240 /*
1241 * limit range
1242 */
1243 if ((n = swap_async_max) > nswbuf / 2)
1244 n = nswbuf / 2;
1245 if (n < 1)
1246 n = 1;
1247 swap_async_max = n;
1248
1249 /*
1250 * Adjust difference ( if possible ). If the current async
1251 * count is too low, we may not be able to make the adjustment
1252 * at this time.
1253 */
1254 s = splvm();
1255 n -= nsw_wcount_async_max;
1256 if (nsw_wcount_async + n >= 0) {
1257 nsw_wcount_async += n;
1258 nsw_wcount_async_max += n;
1259 wakeup(&nsw_wcount_async);
1260 }
1261 splx(s);
1262 }
1263 mtx_unlock(&pbuf_mtx);
1264
1265 /*
1266 * Step 3
1267 *
1268 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1269 * The page is left dirty until the pageout operation completes
1270 * successfully.
1271 */
1272 for (i = 0; i < count; i += n) {
1273 int s;
1274 int j;
1275 struct buf *bp;
1276 daddr_t blk;
1277
1278 /*
1279 * Maximum I/O size is limited by a number of factors.
1280 */
1281 n = min(BLIST_MAX_ALLOC, count - i);
1282 n = min(n, nsw_cluster_max);
1283
1284 s = splvm();
1285
1286 /*
1287 * Get biggest block of swap we can. If we fail, fall
1288 * back and try to allocate a smaller block. Don't go
1289 * overboard trying to allocate space if it would overly
1290 * fragment swap.
1291 */
1292 while (
1293 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1294 n > 4
1295 ) {
1296 n >>= 1;
1297 }
1298 if (blk == SWAPBLK_NONE) {
1299 for (j = 0; j < n; ++j)
1300 rtvals[i+j] = VM_PAGER_FAIL;
1301 splx(s);
1302 continue;
1303 }
1304
1305 /*
1306 * The I/O we are constructing cannot cross a physical
1307 * disk boundry in the swap stripe. Note: we are still
1308 * at splvm().
1309 */
1310 if ((blk ^ (blk + n)) & dmmax_mask) {
1311 j = ((blk + dmmax) & dmmax_mask) - blk;
1312 swp_pager_freeswapspace(blk + j, n - j);
1313 n = j;
1314 }
1315
1316 /*
1317 * All I/O parameters have been satisfied, build the I/O
1318 * request and assign the swap space.
1319 *
1320 * NOTE: B_PAGING is set by pbgetvp()
1321 */
1322 if (sync == TRUE) {
1323 bp = getpbuf(&nsw_wcount_sync);
1324 } else {
1325 bp = getpbuf(&nsw_wcount_async);
1326 bp->b_flags = B_ASYNC;
1327 }
1328 bp->b_iocmd = BIO_WRITE;
1329 bp->b_spc = NULL; /* not used, but NULL-out anyway */
1330
1331 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1332
1333 bp->b_rcred = crhold(thread0.td_ucred);
1334 bp->b_wcred = crhold(thread0.td_ucred);
1335 bp->b_bcount = PAGE_SIZE * n;
1336 bp->b_bufsize = PAGE_SIZE * n;
1337 bp->b_blkno = blk;
1338
1339 pbgetvp(swapdev_vp, bp);
1340
1341 for (j = 0; j < n; ++j) {
1342 vm_page_t mreq = m[i+j];
1343
1344 swp_pager_meta_build(
1345 mreq->object,
1346 mreq->pindex,
1347 blk + j
1348 );
1349 vm_page_dirty(mreq);
1350 rtvals[i+j] = VM_PAGER_OK;
1351
1352 vm_page_flag_set(mreq, PG_SWAPINPROG);
1353 bp->b_pages[j] = mreq;
1354 }
1355 bp->b_npages = n;
1356 /*
1357 * Must set dirty range for NFS to work.
1358 */
1359 bp->b_dirtyoff = 0;
1360 bp->b_dirtyend = bp->b_bcount;
1361
1362 cnt.v_swapout++;
1363 cnt.v_swappgsout += bp->b_npages;
1364 swapdev_vp->v_numoutput++;
1365
1366 splx(s);
1367
1368 /*
1369 * asynchronous
1370 *
1371 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1372 */
1373 if (sync == FALSE) {
1374 bp->b_iodone = swp_pager_async_iodone;
1375 BUF_KERNPROC(bp);
1376 BUF_STRATEGY(bp);
1377
1378 for (j = 0; j < n; ++j)
1379 rtvals[i+j] = VM_PAGER_PEND;
1380 /* restart outter loop */
1381 continue;
1382 }
1383
1384 /*
1385 * synchronous
1386 *
1387 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1388 */
1389 bp->b_iodone = swp_pager_sync_iodone;
1390 BUF_STRATEGY(bp);
1391
1392 /*
1393 * Wait for the sync I/O to complete, then update rtvals.
1394 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1395 * our async completion routine at the end, thus avoiding a
1396 * double-free.
1397 */
1398 s = splbio();
1399 while ((bp->b_flags & B_DONE) == 0) {
1400 tsleep(bp, PVM, "swwrt", 0);
1401 }
1402 for (j = 0; j < n; ++j)
1403 rtvals[i+j] = VM_PAGER_PEND;
1404 /*
1405 * Now that we are through with the bp, we can call the
1406 * normal async completion, which frees everything up.
1407 */
1408 swp_pager_async_iodone(bp);
1409 splx(s);
1410 }
1411}
1412
1413/*
1414 * swap_pager_sync_iodone:
1415 *
1416 * Completion routine for synchronous reads and writes from/to swap.
1417 * We just mark the bp is complete and wake up anyone waiting on it.
1418 *
1419 * This routine may not block. This routine is called at splbio() or better.
1420 */
1421static void
1422swp_pager_sync_iodone(bp)
1423 struct buf *bp;
1424{
1425 bp->b_flags |= B_DONE;
1426 bp->b_flags &= ~B_ASYNC;
1427 wakeup(bp);
1428}
1429
1430/*
1431 * swp_pager_async_iodone:
1432 *
1433 * Completion routine for asynchronous reads and writes from/to swap.
1434 * Also called manually by synchronous code to finish up a bp.
1435 *
1436 * For READ operations, the pages are PG_BUSY'd. For WRITE operations,
1437 * the pages are vm_page_t->busy'd. For READ operations, we PG_BUSY
1438 * unbusy all pages except the 'main' request page. For WRITE
1439 * operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1440 * because we marked them all VM_PAGER_PEND on return from putpages ).
1441 *
1442 * This routine may not block.
1443 * This routine is called at splbio() or better
1444 *
1445 * We up ourselves to splvm() as required for various vm_page related
1446 * calls.
1447 */
1448static void
1449swp_pager_async_iodone(bp)
1450 struct buf *bp;
1451{
1452 int s;
1453 int i;
1454 vm_object_t object = NULL;
1455
1456 GIANT_REQUIRED;
1457 bp->b_flags |= B_DONE;
1458
1459 /*
1460 * report error
1461 */
1462 if (bp->b_ioflags & BIO_ERROR) {
1463 printf(
1464 "swap_pager: I/O error - %s failed; blkno %ld,"
1465 "size %ld, error %d\n",
1466 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1467 (long)bp->b_blkno,
1468 (long)bp->b_bcount,
1469 bp->b_error
1470 );
1471 }
1472
1473 /*
1474 * set object, raise to splvm().
1475 */
1476 if (bp->b_npages)
1477 object = bp->b_pages[0]->object;
1478 s = splvm();
1479
1480 /*
1481 * remove the mapping for kernel virtual
1482 */
1483 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1484
1485 /*
1486 * cleanup pages. If an error occurs writing to swap, we are in
1487 * very serious trouble. If it happens to be a disk error, though,
1488 * we may be able to recover by reassigning the swap later on. So
1489 * in this case we remove the m->swapblk assignment for the page
1490 * but do not free it in the rlist. The errornous block(s) are thus
1491 * never reallocated as swap. Redirty the page and continue.
1492 */
1493 for (i = 0; i < bp->b_npages; ++i) {
1494 vm_page_t m = bp->b_pages[i];
1495
1496 vm_page_flag_clear(m, PG_SWAPINPROG);
1497
1498 if (bp->b_ioflags & BIO_ERROR) {
1499 /*
1500 * If an error occurs I'd love to throw the swapblk
1501 * away without freeing it back to swapspace, so it
1502 * can never be used again. But I can't from an
1503 * interrupt.
1504 */
1505 if (bp->b_iocmd == BIO_READ) {
1506 /*
1507 * When reading, reqpage needs to stay
1508 * locked for the parent, but all other
1509 * pages can be freed. We still want to
1510 * wakeup the parent waiting on the page,
1511 * though. ( also: pg_reqpage can be -1 and
1512 * not match anything ).
1513 *
1514 * We have to wake specifically requested pages
1515 * up too because we cleared PG_SWAPINPROG and
1516 * someone may be waiting for that.
1517 *
1518 * NOTE: for reads, m->dirty will probably
1519 * be overridden by the original caller of
1520 * getpages so don't play cute tricks here.
1521 *
1522 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1523 * AS THIS MESSES WITH object->memq, and it is
1524 * not legal to mess with object->memq from an
1525 * interrupt.
1526 */
1527 m->valid = 0;
1528 vm_page_flag_clear(m, PG_ZERO);
1529 if (i != bp->b_pager.pg_reqpage)
1530 vm_page_free(m);
1531 else
1532 vm_page_flash(m);
1533 /*
1534 * If i == bp->b_pager.pg_reqpage, do not wake
1535 * the page up. The caller needs to.
1536 */
1537 } else {
1538 /*
1539 * If a write error occurs, reactivate page
1540 * so it doesn't clog the inactive list,
1541 * then finish the I/O.
1542 */
1543 vm_page_dirty(m);
1544 vm_page_activate(m);
1545 vm_page_io_finish(m);
1546 }
1547 } else if (bp->b_iocmd == BIO_READ) {
1548 /*
1549 * For read success, clear dirty bits. Nobody should
1550 * have this page mapped but don't take any chances,
1551 * make sure the pmap modify bits are also cleared.
1552 *
1553 * NOTE: for reads, m->dirty will probably be
1554 * overridden by the original caller of getpages so
1555 * we cannot set them in order to free the underlying
1556 * swap in a low-swap situation. I don't think we'd
1557 * want to do that anyway, but it was an optimization
1558 * that existed in the old swapper for a time before
1559 * it got ripped out due to precisely this problem.
1560 *
1561 * clear PG_ZERO in page.
1562 *
1563 * If not the requested page then deactivate it.
1564 *
1565 * Note that the requested page, reqpage, is left
1566 * busied, but we still have to wake it up. The
1567 * other pages are released (unbusied) by
1568 * vm_page_wakeup(). We do not set reqpage's
1569 * valid bits here, it is up to the caller.
1570 */
1571 pmap_clear_modify(m);
1572 m->valid = VM_PAGE_BITS_ALL;
1573 vm_page_undirty(m);
1574 vm_page_flag_clear(m, PG_ZERO);
1575
1576 /*
1577 * We have to wake specifically requested pages
1578 * up too because we cleared PG_SWAPINPROG and
1579 * could be waiting for it in getpages. However,
1580 * be sure to not unbusy getpages specifically
1581 * requested page - getpages expects it to be
1582 * left busy.
1583 */
1584 if (i != bp->b_pager.pg_reqpage) {
1585 vm_page_deactivate(m);
1586 vm_page_wakeup(m);
1587 } else {
1588 vm_page_flash(m);
1589 }
1590 } else {
1591 /*
1592 * For write success, clear the modify and dirty
1593 * status, then finish the I/O ( which decrements the
1594 * busy count and possibly wakes waiter's up ).
1595 */
1596 pmap_clear_modify(m);
1597 vm_page_undirty(m);
1598 vm_page_io_finish(m);
1599 if (!vm_page_count_severe() || !vm_page_try_to_cache(m))
1600 vm_page_protect(m, VM_PROT_READ);
1601 }
1602 }
1603
1604 /*
1605 * adjust pip. NOTE: the original parent may still have its own
1606 * pip refs on the object.
1607 */
1608 if (object)
1609 vm_object_pip_wakeupn(object, bp->b_npages);
1610
1611 /*
1612 * release the physical I/O buffer
1613 */
1614 relpbuf(
1615 bp,
1616 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1617 ((bp->b_flags & B_ASYNC) ?
1618 &nsw_wcount_async :
1619 &nsw_wcount_sync
1620 )
1621 )
1622 );
1623 splx(s);
1624}
1625
1626/************************************************************************
1627 * SWAP META DATA *
1628 ************************************************************************
1629 *
1630 * These routines manipulate the swap metadata stored in the
1631 * OBJT_SWAP object. All swp_*() routines must be called at
1632 * splvm() because swap can be freed up by the low level vm_page
1633 * code which might be called from interrupts beyond what splbio() covers.
1634 *
1635 * Swap metadata is implemented with a global hash and not directly
1636 * linked into the object. Instead the object simply contains
1637 * appropriate tracking counters.
1638 */
1639
1640/*
1641 * SWP_PAGER_HASH() - hash swap meta data
1642 *
1643 * This is an inline helper function which hashes the swapblk given
1644 * the object and page index. It returns a pointer to a pointer
1645 * to the object, or a pointer to a NULL pointer if it could not
1646 * find a swapblk.
1647 *
1648 * This routine must be called at splvm().
1649 */
1650static __inline struct swblock **
1651swp_pager_hash(vm_object_t object, vm_pindex_t index)
1652{
1653 struct swblock **pswap;
1654 struct swblock *swap;
1655
1656 index &= ~SWAP_META_MASK;
1657 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
1658 while ((swap = *pswap) != NULL) {
1659 if (swap->swb_object == object &&
1660 swap->swb_index == index
1661 ) {
1662 break;
1663 }
1664 pswap = &swap->swb_hnext;
1665 }
1666 return (pswap);
1667}
1668
1669/*
1670 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1671 *
1672 * We first convert the object to a swap object if it is a default
1673 * object.
1674 *
1675 * The specified swapblk is added to the object's swap metadata. If
1676 * the swapblk is not valid, it is freed instead. Any previously
1677 * assigned swapblk is freed.
1678 *
1679 * This routine must be called at splvm(), except when used to convert
1680 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1681 */
1682static void
1683swp_pager_meta_build(
1684 vm_object_t object,
1685 vm_pindex_t index,
1686 daddr_t swapblk
1687) {
1688 struct swblock *swap;
1689 struct swblock **pswap;
1690
1691 GIANT_REQUIRED;
1692 /*
1693 * Convert default object to swap object if necessary
1694 */
1695 if (object->type != OBJT_SWAP) {
1696 object->type = OBJT_SWAP;
1697 object->un_pager.swp.swp_bcount = 0;
1698
1699 mtx_lock(&sw_alloc_mtx);
1700 if (object->handle != NULL) {
1701 TAILQ_INSERT_TAIL(
1702 NOBJLIST(object->handle),
1703 object,
1704 pager_object_list
1705 );
1706 } else {
1707 TAILQ_INSERT_TAIL(
1708 &swap_pager_un_object_list,
1709 object,
1710 pager_object_list
1711 );
1712 }
1713 mtx_unlock(&sw_alloc_mtx);
1714 }
1715
1716 /*
1717 * Locate hash entry. If not found create, but if we aren't adding
1718 * anything just return. If we run out of space in the map we wait
1719 * and, since the hash table may have changed, retry.
1720 */
1721retry:
1722 pswap = swp_pager_hash(object, index);
1723
1724 if ((swap = *pswap) == NULL) {
1725 int i;
1726
1727 if (swapblk == SWAPBLK_NONE)
1728 return;
1729
1730 swap = *pswap = zalloc(swap_zone);
1731 if (swap == NULL) {
1732 VM_WAIT;
1733 goto retry;
1734 }
1735 swap->swb_hnext = NULL;
1736 swap->swb_object = object;
1737 swap->swb_index = index & ~SWAP_META_MASK;
1738 swap->swb_count = 0;
1739
1740 ++object->un_pager.swp.swp_bcount;
1741
1742 for (i = 0; i < SWAP_META_PAGES; ++i)
1743 swap->swb_pages[i] = SWAPBLK_NONE;
1744 }
1745
1746 /*
1747 * Delete prior contents of metadata
1748 */
1749 index &= SWAP_META_MASK;
1750
1751 if (swap->swb_pages[index] != SWAPBLK_NONE) {
1752 swp_pager_freeswapspace(swap->swb_pages[index], 1);
1753 --swap->swb_count;
1754 }
1755
1756 /*
1757 * Enter block into metadata
1758 */
1759 swap->swb_pages[index] = swapblk;
1760 if (swapblk != SWAPBLK_NONE)
1761 ++swap->swb_count;
1762}
1763
1764/*
1765 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1766 *
1767 * The requested range of blocks is freed, with any associated swap
1768 * returned to the swap bitmap.
1769 *
1770 * This routine will free swap metadata structures as they are cleaned
1771 * out. This routine does *NOT* operate on swap metadata associated
1772 * with resident pages.
1773 *
1774 * This routine must be called at splvm()
1775 */
1776static void
1777swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1778{
1779 GIANT_REQUIRED;
1780
1781 if (object->type != OBJT_SWAP)
1782 return;
1783
1784 while (count > 0) {
1785 struct swblock **pswap;
1786 struct swblock *swap;
1787
1788 pswap = swp_pager_hash(object, index);
1789
1790 if ((swap = *pswap) != NULL) {
1791 daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1792
1793 if (v != SWAPBLK_NONE) {
1794 swp_pager_freeswapspace(v, 1);
1795 swap->swb_pages[index & SWAP_META_MASK] =
1796 SWAPBLK_NONE;
1797 if (--swap->swb_count == 0) {
1798 *pswap = swap->swb_hnext;
1799 zfree(swap_zone, swap);
1800 --object->un_pager.swp.swp_bcount;
1801 }
1802 }
1803 --count;
1804 ++index;
1805 } else {
1806 int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1807 count -= n;
1808 index += n;
1809 }
1810 }
1811}
1812
1813/*
1814 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1815 *
1816 * This routine locates and destroys all swap metadata associated with
1817 * an object.
1818 *
1819 * This routine must be called at splvm()
1820 */
1821static void
1822swp_pager_meta_free_all(vm_object_t object)
1823{
1824 daddr_t index = 0;
1825
1826 GIANT_REQUIRED;
1827
1828 if (object->type != OBJT_SWAP)
1829 return;
1830
1831 while (object->un_pager.swp.swp_bcount) {
1832 struct swblock **pswap;
1833 struct swblock *swap;
1834
1835 pswap = swp_pager_hash(object, index);
1836 if ((swap = *pswap) != NULL) {
1837 int i;
1838
1839 for (i = 0; i < SWAP_META_PAGES; ++i) {
1840 daddr_t v = swap->swb_pages[i];
1841 if (v != SWAPBLK_NONE) {
1842 --swap->swb_count;
1843 swp_pager_freeswapspace(v, 1);
1844 }
1845 }
1846 if (swap->swb_count != 0)
1847 panic("swap_pager_meta_free_all: swb_count != 0");
1848 *pswap = swap->swb_hnext;
1849 zfree(swap_zone, swap);
1850 --object->un_pager.swp.swp_bcount;
1851 }
1852 index += SWAP_META_PAGES;
1853 if (index > 0x20000000)
1854 panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
1855 }
1856}
1857
1858/*
1859 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data.
1860 *
1861 * This routine is capable of looking up, popping, or freeing
1862 * swapblk assignments in the swap meta data or in the vm_page_t.
1863 * The routine typically returns the swapblk being looked-up, or popped,
1864 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1865 * was invalid. This routine will automatically free any invalid
1866 * meta-data swapblks.
1867 *
1868 * It is not possible to store invalid swapblks in the swap meta data
1869 * (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1870 *
1871 * When acting on a busy resident page and paging is in progress, we
1872 * have to wait until paging is complete but otherwise can act on the
1873 * busy page.
1874 *
1875 * This routine must be called at splvm().
1876 *
1877 * SWM_FREE remove and free swap block from metadata
1878 * SWM_POP remove from meta data but do not free.. pop it out
1879 */
1880static daddr_t
1881swp_pager_meta_ctl(
1882 vm_object_t object,
1883 vm_pindex_t index,
1884 int flags
1885) {
1886 struct swblock **pswap;
1887 struct swblock *swap;
1888 daddr_t r1;
1889
1890 GIANT_REQUIRED;
1891 /*
1892 * The meta data only exists of the object is OBJT_SWAP
1893 * and even then might not be allocated yet.
1894 */
1895 if (object->type != OBJT_SWAP)
1896 return (SWAPBLK_NONE);
1897
1898 r1 = SWAPBLK_NONE;
1899 pswap = swp_pager_hash(object, index);
1900
1901 if ((swap = *pswap) != NULL) {
1902 index &= SWAP_META_MASK;
1903 r1 = swap->swb_pages[index];
1904
1905 if (r1 != SWAPBLK_NONE) {
1906 if (flags & SWM_FREE) {
1907 swp_pager_freeswapspace(r1, 1);
1908 r1 = SWAPBLK_NONE;
1909 }
1910 if (flags & (SWM_FREE|SWM_POP)) {
1911 swap->swb_pages[index] = SWAPBLK_NONE;
1912 if (--swap->swb_count == 0) {
1913 *pswap = swap->swb_hnext;
1914 zfree(swap_zone, swap);
1915 --object->un_pager.swp.swp_bcount;
1916 }
1917 }
1918 }
1919 }
1920 return (r1);
1921}
1922
1923/********************************************************
1924 * CHAINING FUNCTIONS *
1925 ********************************************************
1926 *
1927 * These functions support recursion of I/O operations
1928 * on bp's, typically by chaining one or more 'child' bp's
1929 * to the parent. Synchronous, asynchronous, and semi-synchronous
1930 * chaining is possible.
1931 */
1932
1933/*
1934 * vm_pager_chain_iodone:
1935 *
1936 * io completion routine for child bp. Currently we fudge a bit
1937 * on dealing with b_resid. Since users of these routines may issue
1938 * multiple children simultaneously, sequencing of the error can be lost.
1939 */
1940static void
1941vm_pager_chain_iodone(struct buf *nbp)
1942{
1943 struct bio *bp;
1944 u_int *count;
1945
1946 bp = nbp->b_caller1;
1947 count = (u_int *)&(bp->bio_driver1);
1948 if (bp != NULL) {
1949 if (nbp->b_ioflags & BIO_ERROR) {
1950 bp->bio_flags |= BIO_ERROR;
1951 bp->bio_error = nbp->b_error;
1952 } else if (nbp->b_resid != 0) {
1953 bp->bio_flags |= BIO_ERROR;
1954 bp->bio_error = EINVAL;
1955 } else {
1956 bp->bio_resid -= nbp->b_bcount;
1957 }
1958 nbp->b_caller1 = NULL;
1959 --(*count);
1960 if (bp->bio_flags & BIO_FLAG1) {
1961 bp->bio_flags &= ~BIO_FLAG1;
1962 wakeup(bp);
1963 }
1964 }
1965 nbp->b_flags |= B_DONE;
1966 nbp->b_flags &= ~B_ASYNC;
1967 relpbuf(nbp, NULL);
1968}
1969
1970/*
1971 * getchainbuf:
1972 *
1973 * Obtain a physical buffer and chain it to its parent buffer. When
1974 * I/O completes, the parent buffer will be B_SIGNAL'd. Errors are
1975 * automatically propagated to the parent
1976 */
1977struct buf *
1978getchainbuf(struct bio *bp, struct vnode *vp, int flags)
1979{
1980 struct buf *nbp;
1981 u_int *count;
1982
1983 GIANT_REQUIRED;
1984 nbp = getpbuf(NULL);
1985 count = (u_int *)&(bp->bio_driver1);
1986
1987 nbp->b_caller1 = bp;
1988 ++(*count);
1989
1990 if (*count > 4)
1991 waitchainbuf(bp, 4, 0);
1992
1993 nbp->b_iocmd = bp->bio_cmd;
1994 nbp->b_ioflags = 0;
1995 nbp->b_flags = flags;
1996 nbp->b_rcred = crhold(thread0.td_ucred);
1997 nbp->b_wcred = crhold(thread0.td_ucred);
1998 nbp->b_iodone = vm_pager_chain_iodone;
1999
2000 if (vp)
2001 pbgetvp(vp, nbp);
2002 return (nbp);
2003}
2004
2005void
2006flushchainbuf(struct buf *nbp)
2007{
2008 GIANT_REQUIRED;
2009 if (nbp->b_bcount) {
2010 nbp->b_bufsize = nbp->b_bcount;
2011 if (nbp->b_iocmd == BIO_WRITE)
2012 nbp->b_dirtyend = nbp->b_bcount;
2013 BUF_KERNPROC(nbp);
2014 BUF_STRATEGY(nbp);
2015 } else {
2016 bufdone(nbp);
2017 }
2018}
2019
2020static void
2021waitchainbuf(struct bio *bp, int limit, int done)
2022{
2023 int s;
2024 u_int *count;
2025
2026 GIANT_REQUIRED;
2027 count = (u_int *)&(bp->bio_driver1);
2028 s = splbio();
2029 while (*count > limit) {
2030 bp->bio_flags |= BIO_FLAG1;
2031 tsleep(bp, PRIBIO + 4, "bpchain", 0);
2032 }
2033 if (done) {
2034 if (bp->bio_resid != 0 && !(bp->bio_flags & BIO_ERROR)) {
2035 bp->bio_flags |= BIO_ERROR;
2036 bp->bio_error = EINVAL;
2037 }
2038 biodone(bp);
2039 }
2040 splx(s);
2041}
2042
| 332 break;
333 /*
334 * if the allocation failed, try a zone two thirds the
335 * size of the previous attempt.
336 */
337 n -= ((n + 2) / 3);
338 } while (n > 0);
339 if (swap_zone == NULL)
340 panic("failed to zinit swap_zone.");
341 if (n2 != n)
342 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
343 n2 = n;
344
345 /*
346 * Initialize our meta-data hash table. The swapper does not need to
347 * be quite as efficient as the VM system, so we do not use an
348 * oversized hash table.
349 *
350 * n: size of hash table, must be power of 2
351 * swhash_mask: hash table index mask
352 */
353 for (n = 1; n < n2 / 8; n *= 2)
354 ;
355 swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
356 swhash_mask = n - 1;
357}
358
359/*
360 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
361 * its metadata structures.
362 *
363 * This routine is called from the mmap and fork code to create a new
364 * OBJT_SWAP object. We do this by creating an OBJT_DEFAULT object
365 * and then converting it with swp_pager_meta_build().
366 *
367 * This routine may block in vm_object_allocate() and create a named
368 * object lookup race, so we must interlock. We must also run at
369 * splvm() for the object lookup to handle races with interrupts, but
370 * we do not have to maintain splvm() in between the lookup and the
371 * add because (I believe) it is not possible to attempt to create
372 * a new swap object w/handle when a default object with that handle
373 * already exists.
374 */
375static vm_object_t
376swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
377 vm_ooffset_t offset)
378{
379 vm_object_t object;
380
381 GIANT_REQUIRED;
382
383 if (handle) {
384 /*
385 * Reference existing named region or allocate new one. There
386 * should not be a race here against swp_pager_meta_build()
387 * as called from vm_page_remove() in regards to the lookup
388 * of the handle.
389 */
390 sx_xlock(&sw_alloc_sx);
391 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
392
393 if (object != NULL) {
394 vm_object_reference(object);
395 } else {
396 object = vm_object_allocate(OBJT_DEFAULT,
397 OFF_TO_IDX(offset + PAGE_MASK + size));
398 object->handle = handle;
399
400 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
401 }
402 sx_xunlock(&sw_alloc_sx);
403 } else {
404 object = vm_object_allocate(OBJT_DEFAULT,
405 OFF_TO_IDX(offset + PAGE_MASK + size));
406
407 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
408 }
409
410 return (object);
411}
412
413/*
414 * SWAP_PAGER_DEALLOC() - remove swap metadata from object
415 *
416 * The swap backing for the object is destroyed. The code is
417 * designed such that we can reinstantiate it later, but this
418 * routine is typically called only when the entire object is
419 * about to be destroyed.
420 *
421 * This routine may block, but no longer does.
422 *
423 * The object must be locked or unreferenceable.
424 */
425static void
426swap_pager_dealloc(object)
427 vm_object_t object;
428{
429 int s;
430
431 GIANT_REQUIRED;
432
433 /*
434 * Remove from list right away so lookups will fail if we block for
435 * pageout completion.
436 */
437 mtx_lock(&sw_alloc_mtx);
438 if (object->handle == NULL) {
439 TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list);
440 } else {
441 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
442 }
443 mtx_unlock(&sw_alloc_mtx);
444
445 vm_object_pip_wait(object, "swpdea");
446
447 /*
448 * Free all remaining metadata. We only bother to free it from
449 * the swap meta data. We do not attempt to free swapblk's still
450 * associated with vm_page_t's for this object. We do not care
451 * if paging is still in progress on some objects.
452 */
453 s = splvm();
454 swp_pager_meta_free_all(object);
455 splx(s);
456}
457
458/************************************************************************
459 * SWAP PAGER BITMAP ROUTINES *
460 ************************************************************************/
461
462/*
463 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space
464 *
465 * Allocate swap for the requested number of pages. The starting
466 * swap block number (a page index) is returned or SWAPBLK_NONE
467 * if the allocation failed.
468 *
469 * Also has the side effect of advising that somebody made a mistake
470 * when they configured swap and didn't configure enough.
471 *
472 * Must be called at splvm() to avoid races with bitmap frees from
473 * vm_page_remove() aka swap_pager_page_removed().
474 *
475 * This routine may not block
476 * This routine must be called at splvm().
477 */
478static __inline daddr_t
479swp_pager_getswapspace(npages)
480 int npages;
481{
482 daddr_t blk;
483
484 GIANT_REQUIRED;
485
486 if ((blk = blist_alloc(swapblist, npages)) == SWAPBLK_NONE) {
487 if (swap_pager_full != 2) {
488 printf("swap_pager_getswapspace: failed\n");
489 swap_pager_full = 2;
490 swap_pager_almost_full = 1;
491 }
492 } else {
493 vm_swap_size -= npages;
494 /* per-swap area stats */
495 swdevt[BLK2DEVIDX(blk)].sw_used += npages;
496 swp_sizecheck();
497 }
498 return (blk);
499}
500
501/*
502 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
503 *
504 * This routine returns the specified swap blocks back to the bitmap.
505 *
506 * Note: This routine may not block (it could in the old swap code),
507 * and through the use of the new blist routines it does not block.
508 *
509 * We must be called at splvm() to avoid races with bitmap frees from
510 * vm_page_remove() aka swap_pager_page_removed().
511 *
512 * This routine may not block
513 * This routine must be called at splvm().
514 */
515static __inline void
516swp_pager_freeswapspace(blk, npages)
517 daddr_t blk;
518 int npages;
519{
520 GIANT_REQUIRED;
521
522 blist_free(swapblist, blk, npages);
523 vm_swap_size += npages;
524 /* per-swap area stats */
525 swdevt[BLK2DEVIDX(blk)].sw_used -= npages;
526 swp_sizecheck();
527}
528
529/*
530 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
531 * range within an object.
532 *
533 * This is a globally accessible routine.
534 *
535 * This routine removes swapblk assignments from swap metadata.
536 *
537 * The external callers of this routine typically have already destroyed
538 * or renamed vm_page_t's associated with this range in the object so
539 * we should be ok.
540 *
541 * This routine may be called at any spl. We up our spl to splvm temporarily
542 * in order to perform the metadata removal.
543 */
544void
545swap_pager_freespace(object, start, size)
546 vm_object_t object;
547 vm_pindex_t start;
548 vm_size_t size;
549{
550 int s = splvm();
551
552 GIANT_REQUIRED;
553 swp_pager_meta_free(object, start, size);
554 splx(s);
555}
556
557/*
558 * SWAP_PAGER_RESERVE() - reserve swap blocks in object
559 *
560 * Assigns swap blocks to the specified range within the object. The
561 * swap blocks are not zerod. Any previous swap assignment is destroyed.
562 *
563 * Returns 0 on success, -1 on failure.
564 */
565int
566swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
567{
568 int s;
569 int n = 0;
570 daddr_t blk = SWAPBLK_NONE;
571 vm_pindex_t beg = start; /* save start index */
572
573 s = splvm();
574 while (size) {
575 if (n == 0) {
576 n = BLIST_MAX_ALLOC;
577 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
578 n >>= 1;
579 if (n == 0) {
580 swp_pager_meta_free(object, beg, start - beg);
581 splx(s);
582 return (-1);
583 }
584 }
585 }
586 swp_pager_meta_build(object, start, blk);
587 --size;
588 ++start;
589 ++blk;
590 --n;
591 }
592 swp_pager_meta_free(object, start, n);
593 splx(s);
594 return (0);
595}
596
597/*
598 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
599 * and destroy the source.
600 *
601 * Copy any valid swapblks from the source to the destination. In
602 * cases where both the source and destination have a valid swapblk,
603 * we keep the destination's.
604 *
605 * This routine is allowed to block. It may block allocating metadata
606 * indirectly through swp_pager_meta_build() or if paging is still in
607 * progress on the source.
608 *
609 * This routine can be called at any spl
610 *
611 * XXX vm_page_collapse() kinda expects us not to block because we
612 * supposedly do not need to allocate memory, but for the moment we
613 * *may* have to get a little memory from the zone allocator, but
614 * it is taken from the interrupt memory. We should be ok.
615 *
616 * The source object contains no vm_page_t's (which is just as well)
617 *
618 * The source object is of type OBJT_SWAP.
619 *
620 * The source and destination objects must be locked or
621 * inaccessible (XXX are they ?)
622 */
623void
624swap_pager_copy(srcobject, dstobject, offset, destroysource)
625 vm_object_t srcobject;
626 vm_object_t dstobject;
627 vm_pindex_t offset;
628 int destroysource;
629{
630 vm_pindex_t i;
631 int s;
632
633 GIANT_REQUIRED;
634
635 s = splvm();
636 /*
637 * If destroysource is set, we remove the source object from the
638 * swap_pager internal queue now.
639 */
640 if (destroysource) {
641 mtx_lock(&sw_alloc_mtx);
642 if (srcobject->handle == NULL) {
643 TAILQ_REMOVE(
644 &swap_pager_un_object_list,
645 srcobject,
646 pager_object_list
647 );
648 } else {
649 TAILQ_REMOVE(
650 NOBJLIST(srcobject->handle),
651 srcobject,
652 pager_object_list
653 );
654 }
655 mtx_unlock(&sw_alloc_mtx);
656 }
657
658 /*
659 * transfer source to destination.
660 */
661 for (i = 0; i < dstobject->size; ++i) {
662 daddr_t dstaddr;
663
664 /*
665 * Locate (without changing) the swapblk on the destination,
666 * unless it is invalid in which case free it silently, or
667 * if the destination is a resident page, in which case the
668 * source is thrown away.
669 */
670 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
671
672 if (dstaddr == SWAPBLK_NONE) {
673 /*
674 * Destination has no swapblk and is not resident,
675 * copy source.
676 */
677 daddr_t srcaddr;
678
679 srcaddr = swp_pager_meta_ctl(
680 srcobject,
681 i + offset,
682 SWM_POP
683 );
684
685 if (srcaddr != SWAPBLK_NONE)
686 swp_pager_meta_build(dstobject, i, srcaddr);
687 } else {
688 /*
689 * Destination has valid swapblk or it is represented
690 * by a resident page. We destroy the sourceblock.
691 */
692
693 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
694 }
695 }
696
697 /*
698 * Free left over swap blocks in source.
699 *
700 * We have to revert the type to OBJT_DEFAULT so we do not accidently
701 * double-remove the object from the swap queues.
702 */
703 if (destroysource) {
704 swp_pager_meta_free_all(srcobject);
705 /*
706 * Reverting the type is not necessary, the caller is going
707 * to destroy srcobject directly, but I'm doing it here
708 * for consistency since we've removed the object from its
709 * queues.
710 */
711 srcobject->type = OBJT_DEFAULT;
712 }
713 splx(s);
714}
715
716/*
717 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for
718 * the requested page.
719 *
720 * We determine whether good backing store exists for the requested
721 * page and return TRUE if it does, FALSE if it doesn't.
722 *
723 * If TRUE, we also try to determine how much valid, contiguous backing
724 * store exists before and after the requested page within a reasonable
725 * distance. We do not try to restrict it to the swap device stripe
726 * (that is handled in getpages/putpages). It probably isn't worth
727 * doing here.
728 */
729boolean_t
730swap_pager_haspage(object, pindex, before, after)
731 vm_object_t object;
732 vm_pindex_t pindex;
733 int *before;
734 int *after;
735{
736 daddr_t blk0;
737 int s;
738
739 /*
740 * do we have good backing store at the requested index ?
741 */
742 s = splvm();
743 blk0 = swp_pager_meta_ctl(object, pindex, 0);
744
745 if (blk0 == SWAPBLK_NONE) {
746 splx(s);
747 if (before)
748 *before = 0;
749 if (after)
750 *after = 0;
751 return (FALSE);
752 }
753
754 /*
755 * find backwards-looking contiguous good backing store
756 */
757 if (before != NULL) {
758 int i;
759
760 for (i = 1; i < (SWB_NPAGES/2); ++i) {
761 daddr_t blk;
762
763 if (i > pindex)
764 break;
765 blk = swp_pager_meta_ctl(object, pindex - i, 0);
766 if (blk != blk0 - i)
767 break;
768 }
769 *before = (i - 1);
770 }
771
772 /*
773 * find forward-looking contiguous good backing store
774 */
775 if (after != NULL) {
776 int i;
777
778 for (i = 1; i < (SWB_NPAGES/2); ++i) {
779 daddr_t blk;
780
781 blk = swp_pager_meta_ctl(object, pindex + i, 0);
782 if (blk != blk0 + i)
783 break;
784 }
785 *after = (i - 1);
786 }
787 splx(s);
788 return (TRUE);
789}
790
791/*
792 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
793 *
794 * This removes any associated swap backing store, whether valid or
795 * not, from the page.
796 *
797 * This routine is typically called when a page is made dirty, at
798 * which point any associated swap can be freed. MADV_FREE also
799 * calls us in a special-case situation
800 *
801 * NOTE!!! If the page is clean and the swap was valid, the caller
802 * should make the page dirty before calling this routine. This routine
803 * does NOT change the m->dirty status of the page. Also: MADV_FREE
804 * depends on it.
805 *
806 * This routine may not block
807 * This routine must be called at splvm()
808 */
809static void
810swap_pager_unswapped(m)
811 vm_page_t m;
812{
813 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
814}
815
816/*
817 * SWAP_PAGER_STRATEGY() - read, write, free blocks
818 *
819 * This implements the vm_pager_strategy() interface to swap and allows
820 * other parts of the system to directly access swap as backing store
821 * through vm_objects of type OBJT_SWAP. This is intended to be a
822 * cacheless interface ( i.e. caching occurs at higher levels ).
823 * Therefore we do not maintain any resident pages. All I/O goes
824 * directly to and from the swap device.
825 *
826 * Note that b_blkno is scaled for PAGE_SIZE
827 *
828 * We currently attempt to run I/O synchronously or asynchronously as
829 * the caller requests. This isn't perfect because we loose error
830 * sequencing when we run multiple ops in parallel to satisfy a request.
831 * But this is swap, so we let it all hang out.
832 */
833static void
834swap_pager_strategy(vm_object_t object, struct bio *bp)
835{
836 vm_pindex_t start;
837 int count;
838 int s;
839 char *data;
840 struct buf *nbp = NULL;
841
842 GIANT_REQUIRED;
843
844 /* XXX: KASSERT instead ? */
845 if (bp->bio_bcount & PAGE_MASK) {
846 biofinish(bp, NULL, EINVAL);
847 printf("swap_pager_strategy: bp %p blk %d size %d, not page bounded\n", bp, (int)bp->bio_pblkno, (int)bp->bio_bcount);
848 return;
849 }
850
851 /*
852 * Clear error indication, initialize page index, count, data pointer.
853 */
854 bp->bio_error = 0;
855 bp->bio_flags &= ~BIO_ERROR;
856 bp->bio_resid = bp->bio_bcount;
857 *(u_int *) &bp->bio_driver1 = 0;
858
859 start = bp->bio_pblkno;
860 count = howmany(bp->bio_bcount, PAGE_SIZE);
861 data = bp->bio_data;
862
863 s = splvm();
864
865 /*
866 * Deal with BIO_DELETE
867 */
868 if (bp->bio_cmd == BIO_DELETE) {
869 /*
870 * FREE PAGE(s) - destroy underlying swap that is no longer
871 * needed.
872 */
873 swp_pager_meta_free(object, start, count);
874 splx(s);
875 bp->bio_resid = 0;
876 biodone(bp);
877 return;
878 }
879
880 /*
881 * Execute read or write
882 */
883 while (count > 0) {
884 daddr_t blk;
885
886 /*
887 * Obtain block. If block not found and writing, allocate a
888 * new block and build it into the object.
889 */
890
891 blk = swp_pager_meta_ctl(object, start, 0);
892 if ((blk == SWAPBLK_NONE) && (bp->bio_cmd == BIO_WRITE)) {
893 blk = swp_pager_getswapspace(1);
894 if (blk == SWAPBLK_NONE) {
895 bp->bio_error = ENOMEM;
896 bp->bio_flags |= BIO_ERROR;
897 break;
898 }
899 swp_pager_meta_build(object, start, blk);
900 }
901
902 /*
903 * Do we have to flush our current collection? Yes if:
904 *
905 * - no swap block at this index
906 * - swap block is not contiguous
907 * - we cross a physical disk boundry in the
908 * stripe.
909 */
910 if (
911 nbp && (nbp->b_blkno + btoc(nbp->b_bcount) != blk ||
912 ((nbp->b_blkno ^ blk) & dmmax_mask)
913 )
914 ) {
915 splx(s);
916 if (bp->bio_cmd == BIO_READ) {
917 ++cnt.v_swapin;
918 cnt.v_swappgsin += btoc(nbp->b_bcount);
919 } else {
920 ++cnt.v_swapout;
921 cnt.v_swappgsout += btoc(nbp->b_bcount);
922 nbp->b_dirtyend = nbp->b_bcount;
923 }
924 flushchainbuf(nbp);
925 s = splvm();
926 nbp = NULL;
927 }
928
929 /*
930 * Add new swapblk to nbp, instantiating nbp if necessary.
931 * Zero-fill reads are able to take a shortcut.
932 */
933 if (blk == SWAPBLK_NONE) {
934 /*
935 * We can only get here if we are reading. Since
936 * we are at splvm() we can safely modify b_resid,
937 * even if chain ops are in progress.
938 */
939 bzero(data, PAGE_SIZE);
940 bp->bio_resid -= PAGE_SIZE;
941 } else {
942 if (nbp == NULL) {
943 nbp = getchainbuf(bp, swapdev_vp, B_ASYNC);
944 nbp->b_blkno = blk;
945 nbp->b_bcount = 0;
946 nbp->b_data = data;
947 }
948 nbp->b_bcount += PAGE_SIZE;
949 }
950 --count;
951 ++start;
952 data += PAGE_SIZE;
953 }
954
955 /*
956 * Flush out last buffer
957 */
958 splx(s);
959
960 if (nbp) {
961 if (nbp->b_iocmd == BIO_READ) {
962 ++cnt.v_swapin;
963 cnt.v_swappgsin += btoc(nbp->b_bcount);
964 } else {
965 ++cnt.v_swapout;
966 cnt.v_swappgsout += btoc(nbp->b_bcount);
967 nbp->b_dirtyend = nbp->b_bcount;
968 }
969 flushchainbuf(nbp);
970 /* nbp = NULL; */
971 }
972 /*
973 * Wait for completion.
974 */
975 waitchainbuf(bp, 0, 1);
976}
977
978/*
979 * SWAP_PAGER_GETPAGES() - bring pages in from swap
980 *
981 * Attempt to retrieve (m, count) pages from backing store, but make
982 * sure we retrieve at least m[reqpage]. We try to load in as large
983 * a chunk surrounding m[reqpage] as is contiguous in swap and which
984 * belongs to the same object.
985 *
986 * The code is designed for asynchronous operation and
987 * immediate-notification of 'reqpage' but tends not to be
988 * used that way. Please do not optimize-out this algorithmic
989 * feature, I intend to improve on it in the future.
990 *
991 * The parent has a single vm_object_pip_add() reference prior to
992 * calling us and we should return with the same.
993 *
994 * The parent has BUSY'd the pages. We should return with 'm'
995 * left busy, but the others adjusted.
996 */
997static int
998swap_pager_getpages(object, m, count, reqpage)
999 vm_object_t object;
1000 vm_page_t *m;
1001 int count, reqpage;
1002{
1003 struct buf *bp;
1004 vm_page_t mreq;
1005 int s;
1006 int i;
1007 int j;
1008 daddr_t blk;
1009 vm_offset_t kva;
1010 vm_pindex_t lastpindex;
1011
1012 GIANT_REQUIRED;
1013
1014 mreq = m[reqpage];
1015
1016 if (mreq->object != object) {
1017 panic("swap_pager_getpages: object mismatch %p/%p",
1018 object,
1019 mreq->object
1020 );
1021 }
1022 /*
1023 * Calculate range to retrieve. The pages have already been assigned
1024 * their swapblks. We require a *contiguous* range that falls entirely
1025 * within a single device stripe. If we do not supply it, bad things
1026 * happen. Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
1027 * loops are set up such that the case(s) are handled implicitly.
1028 *
1029 * The swp_*() calls must be made at splvm(). vm_page_free() does
1030 * not need to be, but it will go a little faster if it is.
1031 */
1032 s = splvm();
1033 blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
1034
1035 for (i = reqpage - 1; i >= 0; --i) {
1036 daddr_t iblk;
1037
1038 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
1039 if (blk != iblk + (reqpage - i))
1040 break;
1041 if ((blk ^ iblk) & dmmax_mask)
1042 break;
1043 }
1044 ++i;
1045
1046 for (j = reqpage + 1; j < count; ++j) {
1047 daddr_t jblk;
1048
1049 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
1050 if (blk != jblk - (j - reqpage))
1051 break;
1052 if ((blk ^ jblk) & dmmax_mask)
1053 break;
1054 }
1055
1056 /*
1057 * free pages outside our collection range. Note: we never free
1058 * mreq, it must remain busy throughout.
1059 */
1060 {
1061 int k;
1062
1063 for (k = 0; k < i; ++k)
1064 vm_page_free(m[k]);
1065 for (k = j; k < count; ++k)
1066 vm_page_free(m[k]);
1067 }
1068 splx(s);
1069
1070
1071 /*
1072 * Return VM_PAGER_FAIL if we have nothing to do. Return mreq
1073 * still busy, but the others unbusied.
1074 */
1075 if (blk == SWAPBLK_NONE)
1076 return (VM_PAGER_FAIL);
1077
1078 /*
1079 * Get a swap buffer header to perform the IO
1080 */
1081 bp = getpbuf(&nsw_rcount);
1082 kva = (vm_offset_t) bp->b_data;
1083
1084 /*
1085 * map our page(s) into kva for input
1086 *
1087 * NOTE: B_PAGING is set by pbgetvp()
1088 */
1089 pmap_qenter(kva, m + i, j - i);
1090
1091 bp->b_iocmd = BIO_READ;
1092 bp->b_iodone = swp_pager_async_iodone;
1093 bp->b_rcred = crhold(thread0.td_ucred);
1094 bp->b_wcred = crhold(thread0.td_ucred);
1095 bp->b_data = (caddr_t) kva;
1096 bp->b_blkno = blk - (reqpage - i);
1097 bp->b_bcount = PAGE_SIZE * (j - i);
1098 bp->b_bufsize = PAGE_SIZE * (j - i);
1099 bp->b_pager.pg_reqpage = reqpage - i;
1100
1101 {
1102 int k;
1103
1104 for (k = i; k < j; ++k) {
1105 bp->b_pages[k - i] = m[k];
1106 vm_page_flag_set(m[k], PG_SWAPINPROG);
1107 }
1108 }
1109 bp->b_npages = j - i;
1110
1111 pbgetvp(swapdev_vp, bp);
1112
1113 cnt.v_swapin++;
1114 cnt.v_swappgsin += bp->b_npages;
1115
1116 /*
1117 * We still hold the lock on mreq, and our automatic completion routine
1118 * does not remove it.
1119 */
1120 vm_object_pip_add(mreq->object, bp->b_npages);
1121 lastpindex = m[j-1]->pindex;
1122
1123 /*
1124 * perform the I/O. NOTE!!! bp cannot be considered valid after
1125 * this point because we automatically release it on completion.
1126 * Instead, we look at the one page we are interested in which we
1127 * still hold a lock on even through the I/O completion.
1128 *
1129 * The other pages in our m[] array are also released on completion,
1130 * so we cannot assume they are valid anymore either.
1131 *
1132 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1133 */
1134 BUF_KERNPROC(bp);
1135 BUF_STRATEGY(bp);
1136
1137 /*
1138 * wait for the page we want to complete. PG_SWAPINPROG is always
1139 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE
1140 * is set in the meta-data.
1141 */
1142 s = splvm();
1143 while ((mreq->flags & PG_SWAPINPROG) != 0) {
1144 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
1145 cnt.v_intrans++;
1146 if (tsleep(mreq, PSWP, "swread", hz*20)) {
1147 printf(
1148 "swap_pager: indefinite wait buffer: device:"
1149 " %s, blkno: %ld, size: %ld\n",
1150 devtoname(bp->b_dev), (long)bp->b_blkno,
1151 bp->b_bcount
1152 );
1153 }
1154 }
1155 splx(s);
1156
1157 /*
1158 * mreq is left busied after completion, but all the other pages
1159 * are freed. If we had an unrecoverable read error the page will
1160 * not be valid.
1161 */
1162 if (mreq->valid != VM_PAGE_BITS_ALL) {
1163 return (VM_PAGER_ERROR);
1164 } else {
1165 return (VM_PAGER_OK);
1166 }
1167
1168 /*
1169 * A final note: in a low swap situation, we cannot deallocate swap
1170 * and mark a page dirty here because the caller is likely to mark
1171 * the page clean when we return, causing the page to possibly revert
1172 * to all-zero's later.
1173 */
1174}
1175
1176/*
1177 * swap_pager_putpages:
1178 *
1179 * Assign swap (if necessary) and initiate I/O on the specified pages.
1180 *
1181 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects
1182 * are automatically converted to SWAP objects.
1183 *
1184 * In a low memory situation we may block in VOP_STRATEGY(), but the new
1185 * vm_page reservation system coupled with properly written VFS devices
1186 * should ensure that no low-memory deadlock occurs. This is an area
1187 * which needs work.
1188 *
1189 * The parent has N vm_object_pip_add() references prior to
1190 * calling us and will remove references for rtvals[] that are
1191 * not set to VM_PAGER_PEND. We need to remove the rest on I/O
1192 * completion.
1193 *
1194 * The parent has soft-busy'd the pages it passes us and will unbusy
1195 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
1196 * We need to unbusy the rest on I/O completion.
1197 */
1198void
1199swap_pager_putpages(object, m, count, sync, rtvals)
1200 vm_object_t object;
1201 vm_page_t *m;
1202 int count;
1203 boolean_t sync;
1204 int *rtvals;
1205{
1206 int i;
1207 int n = 0;
1208
1209 GIANT_REQUIRED;
1210 if (count && m[0]->object != object) {
1211 panic("swap_pager_getpages: object mismatch %p/%p",
1212 object,
1213 m[0]->object
1214 );
1215 }
1216 /*
1217 * Step 1
1218 *
1219 * Turn object into OBJT_SWAP
1220 * check for bogus sysops
1221 * force sync if not pageout process
1222 */
1223 if (object->type != OBJT_SWAP)
1224 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
1225
1226 if (curproc != pageproc)
1227 sync = TRUE;
1228
1229 /*
1230 * Step 2
1231 *
1232 * Update nsw parameters from swap_async_max sysctl values.
1233 * Do not let the sysop crash the machine with bogus numbers.
1234 */
1235 mtx_lock(&pbuf_mtx);
1236 if (swap_async_max != nsw_wcount_async_max) {
1237 int n;
1238 int s;
1239
1240 /*
1241 * limit range
1242 */
1243 if ((n = swap_async_max) > nswbuf / 2)
1244 n = nswbuf / 2;
1245 if (n < 1)
1246 n = 1;
1247 swap_async_max = n;
1248
1249 /*
1250 * Adjust difference ( if possible ). If the current async
1251 * count is too low, we may not be able to make the adjustment
1252 * at this time.
1253 */
1254 s = splvm();
1255 n -= nsw_wcount_async_max;
1256 if (nsw_wcount_async + n >= 0) {
1257 nsw_wcount_async += n;
1258 nsw_wcount_async_max += n;
1259 wakeup(&nsw_wcount_async);
1260 }
1261 splx(s);
1262 }
1263 mtx_unlock(&pbuf_mtx);
1264
1265 /*
1266 * Step 3
1267 *
1268 * Assign swap blocks and issue I/O. We reallocate swap on the fly.
1269 * The page is left dirty until the pageout operation completes
1270 * successfully.
1271 */
1272 for (i = 0; i < count; i += n) {
1273 int s;
1274 int j;
1275 struct buf *bp;
1276 daddr_t blk;
1277
1278 /*
1279 * Maximum I/O size is limited by a number of factors.
1280 */
1281 n = min(BLIST_MAX_ALLOC, count - i);
1282 n = min(n, nsw_cluster_max);
1283
1284 s = splvm();
1285
1286 /*
1287 * Get biggest block of swap we can. If we fail, fall
1288 * back and try to allocate a smaller block. Don't go
1289 * overboard trying to allocate space if it would overly
1290 * fragment swap.
1291 */
1292 while (
1293 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
1294 n > 4
1295 ) {
1296 n >>= 1;
1297 }
1298 if (blk == SWAPBLK_NONE) {
1299 for (j = 0; j < n; ++j)
1300 rtvals[i+j] = VM_PAGER_FAIL;
1301 splx(s);
1302 continue;
1303 }
1304
1305 /*
1306 * The I/O we are constructing cannot cross a physical
1307 * disk boundry in the swap stripe. Note: we are still
1308 * at splvm().
1309 */
1310 if ((blk ^ (blk + n)) & dmmax_mask) {
1311 j = ((blk + dmmax) & dmmax_mask) - blk;
1312 swp_pager_freeswapspace(blk + j, n - j);
1313 n = j;
1314 }
1315
1316 /*
1317 * All I/O parameters have been satisfied, build the I/O
1318 * request and assign the swap space.
1319 *
1320 * NOTE: B_PAGING is set by pbgetvp()
1321 */
1322 if (sync == TRUE) {
1323 bp = getpbuf(&nsw_wcount_sync);
1324 } else {
1325 bp = getpbuf(&nsw_wcount_async);
1326 bp->b_flags = B_ASYNC;
1327 }
1328 bp->b_iocmd = BIO_WRITE;
1329 bp->b_spc = NULL; /* not used, but NULL-out anyway */
1330
1331 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
1332
1333 bp->b_rcred = crhold(thread0.td_ucred);
1334 bp->b_wcred = crhold(thread0.td_ucred);
1335 bp->b_bcount = PAGE_SIZE * n;
1336 bp->b_bufsize = PAGE_SIZE * n;
1337 bp->b_blkno = blk;
1338
1339 pbgetvp(swapdev_vp, bp);
1340
1341 for (j = 0; j < n; ++j) {
1342 vm_page_t mreq = m[i+j];
1343
1344 swp_pager_meta_build(
1345 mreq->object,
1346 mreq->pindex,
1347 blk + j
1348 );
1349 vm_page_dirty(mreq);
1350 rtvals[i+j] = VM_PAGER_OK;
1351
1352 vm_page_flag_set(mreq, PG_SWAPINPROG);
1353 bp->b_pages[j] = mreq;
1354 }
1355 bp->b_npages = n;
1356 /*
1357 * Must set dirty range for NFS to work.
1358 */
1359 bp->b_dirtyoff = 0;
1360 bp->b_dirtyend = bp->b_bcount;
1361
1362 cnt.v_swapout++;
1363 cnt.v_swappgsout += bp->b_npages;
1364 swapdev_vp->v_numoutput++;
1365
1366 splx(s);
1367
1368 /*
1369 * asynchronous
1370 *
1371 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1372 */
1373 if (sync == FALSE) {
1374 bp->b_iodone = swp_pager_async_iodone;
1375 BUF_KERNPROC(bp);
1376 BUF_STRATEGY(bp);
1377
1378 for (j = 0; j < n; ++j)
1379 rtvals[i+j] = VM_PAGER_PEND;
1380 /* restart outter loop */
1381 continue;
1382 }
1383
1384 /*
1385 * synchronous
1386 *
1387 * NOTE: b_blkno is destroyed by the call to VOP_STRATEGY
1388 */
1389 bp->b_iodone = swp_pager_sync_iodone;
1390 BUF_STRATEGY(bp);
1391
1392 /*
1393 * Wait for the sync I/O to complete, then update rtvals.
1394 * We just set the rtvals[] to VM_PAGER_PEND so we can call
1395 * our async completion routine at the end, thus avoiding a
1396 * double-free.
1397 */
1398 s = splbio();
1399 while ((bp->b_flags & B_DONE) == 0) {
1400 tsleep(bp, PVM, "swwrt", 0);
1401 }
1402 for (j = 0; j < n; ++j)
1403 rtvals[i+j] = VM_PAGER_PEND;
1404 /*
1405 * Now that we are through with the bp, we can call the
1406 * normal async completion, which frees everything up.
1407 */
1408 swp_pager_async_iodone(bp);
1409 splx(s);
1410 }
1411}
1412
1413/*
1414 * swap_pager_sync_iodone:
1415 *
1416 * Completion routine for synchronous reads and writes from/to swap.
1417 * We just mark the bp is complete and wake up anyone waiting on it.
1418 *
1419 * This routine may not block. This routine is called at splbio() or better.
1420 */
1421static void
1422swp_pager_sync_iodone(bp)
1423 struct buf *bp;
1424{
1425 bp->b_flags |= B_DONE;
1426 bp->b_flags &= ~B_ASYNC;
1427 wakeup(bp);
1428}
1429
1430/*
1431 * swp_pager_async_iodone:
1432 *
1433 * Completion routine for asynchronous reads and writes from/to swap.
1434 * Also called manually by synchronous code to finish up a bp.
1435 *
1436 * For READ operations, the pages are PG_BUSY'd. For WRITE operations,
1437 * the pages are vm_page_t->busy'd. For READ operations, we PG_BUSY
1438 * unbusy all pages except the 'main' request page. For WRITE
1439 * operations, we vm_page_t->busy'd unbusy all pages ( we can do this
1440 * because we marked them all VM_PAGER_PEND on return from putpages ).
1441 *
1442 * This routine may not block.
1443 * This routine is called at splbio() or better
1444 *
1445 * We up ourselves to splvm() as required for various vm_page related
1446 * calls.
1447 */
1448static void
1449swp_pager_async_iodone(bp)
1450 struct buf *bp;
1451{
1452 int s;
1453 int i;
1454 vm_object_t object = NULL;
1455
1456 GIANT_REQUIRED;
1457 bp->b_flags |= B_DONE;
1458
1459 /*
1460 * report error
1461 */
1462 if (bp->b_ioflags & BIO_ERROR) {
1463 printf(
1464 "swap_pager: I/O error - %s failed; blkno %ld,"
1465 "size %ld, error %d\n",
1466 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
1467 (long)bp->b_blkno,
1468 (long)bp->b_bcount,
1469 bp->b_error
1470 );
1471 }
1472
1473 /*
1474 * set object, raise to splvm().
1475 */
1476 if (bp->b_npages)
1477 object = bp->b_pages[0]->object;
1478 s = splvm();
1479
1480 /*
1481 * remove the mapping for kernel virtual
1482 */
1483 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
1484
1485 /*
1486 * cleanup pages. If an error occurs writing to swap, we are in
1487 * very serious trouble. If it happens to be a disk error, though,
1488 * we may be able to recover by reassigning the swap later on. So
1489 * in this case we remove the m->swapblk assignment for the page
1490 * but do not free it in the rlist. The errornous block(s) are thus
1491 * never reallocated as swap. Redirty the page and continue.
1492 */
1493 for (i = 0; i < bp->b_npages; ++i) {
1494 vm_page_t m = bp->b_pages[i];
1495
1496 vm_page_flag_clear(m, PG_SWAPINPROG);
1497
1498 if (bp->b_ioflags & BIO_ERROR) {
1499 /*
1500 * If an error occurs I'd love to throw the swapblk
1501 * away without freeing it back to swapspace, so it
1502 * can never be used again. But I can't from an
1503 * interrupt.
1504 */
1505 if (bp->b_iocmd == BIO_READ) {
1506 /*
1507 * When reading, reqpage needs to stay
1508 * locked for the parent, but all other
1509 * pages can be freed. We still want to
1510 * wakeup the parent waiting on the page,
1511 * though. ( also: pg_reqpage can be -1 and
1512 * not match anything ).
1513 *
1514 * We have to wake specifically requested pages
1515 * up too because we cleared PG_SWAPINPROG and
1516 * someone may be waiting for that.
1517 *
1518 * NOTE: for reads, m->dirty will probably
1519 * be overridden by the original caller of
1520 * getpages so don't play cute tricks here.
1521 *
1522 * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
1523 * AS THIS MESSES WITH object->memq, and it is
1524 * not legal to mess with object->memq from an
1525 * interrupt.
1526 */
1527 m->valid = 0;
1528 vm_page_flag_clear(m, PG_ZERO);
1529 if (i != bp->b_pager.pg_reqpage)
1530 vm_page_free(m);
1531 else
1532 vm_page_flash(m);
1533 /*
1534 * If i == bp->b_pager.pg_reqpage, do not wake
1535 * the page up. The caller needs to.
1536 */
1537 } else {
1538 /*
1539 * If a write error occurs, reactivate page
1540 * so it doesn't clog the inactive list,
1541 * then finish the I/O.
1542 */
1543 vm_page_dirty(m);
1544 vm_page_activate(m);
1545 vm_page_io_finish(m);
1546 }
1547 } else if (bp->b_iocmd == BIO_READ) {
1548 /*
1549 * For read success, clear dirty bits. Nobody should
1550 * have this page mapped but don't take any chances,
1551 * make sure the pmap modify bits are also cleared.
1552 *
1553 * NOTE: for reads, m->dirty will probably be
1554 * overridden by the original caller of getpages so
1555 * we cannot set them in order to free the underlying
1556 * swap in a low-swap situation. I don't think we'd
1557 * want to do that anyway, but it was an optimization
1558 * that existed in the old swapper for a time before
1559 * it got ripped out due to precisely this problem.
1560 *
1561 * clear PG_ZERO in page.
1562 *
1563 * If not the requested page then deactivate it.
1564 *
1565 * Note that the requested page, reqpage, is left
1566 * busied, but we still have to wake it up. The
1567 * other pages are released (unbusied) by
1568 * vm_page_wakeup(). We do not set reqpage's
1569 * valid bits here, it is up to the caller.
1570 */
1571 pmap_clear_modify(m);
1572 m->valid = VM_PAGE_BITS_ALL;
1573 vm_page_undirty(m);
1574 vm_page_flag_clear(m, PG_ZERO);
1575
1576 /*
1577 * We have to wake specifically requested pages
1578 * up too because we cleared PG_SWAPINPROG and
1579 * could be waiting for it in getpages. However,
1580 * be sure to not unbusy getpages specifically
1581 * requested page - getpages expects it to be
1582 * left busy.
1583 */
1584 if (i != bp->b_pager.pg_reqpage) {
1585 vm_page_deactivate(m);
1586 vm_page_wakeup(m);
1587 } else {
1588 vm_page_flash(m);
1589 }
1590 } else {
1591 /*
1592 * For write success, clear the modify and dirty
1593 * status, then finish the I/O ( which decrements the
1594 * busy count and possibly wakes waiter's up ).
1595 */
1596 pmap_clear_modify(m);
1597 vm_page_undirty(m);
1598 vm_page_io_finish(m);
1599 if (!vm_page_count_severe() || !vm_page_try_to_cache(m))
1600 vm_page_protect(m, VM_PROT_READ);
1601 }
1602 }
1603
1604 /*
1605 * adjust pip. NOTE: the original parent may still have its own
1606 * pip refs on the object.
1607 */
1608 if (object)
1609 vm_object_pip_wakeupn(object, bp->b_npages);
1610
1611 /*
1612 * release the physical I/O buffer
1613 */
1614 relpbuf(
1615 bp,
1616 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
1617 ((bp->b_flags & B_ASYNC) ?
1618 &nsw_wcount_async :
1619 &nsw_wcount_sync
1620 )
1621 )
1622 );
1623 splx(s);
1624}
1625
1626/************************************************************************
1627 * SWAP META DATA *
1628 ************************************************************************
1629 *
1630 * These routines manipulate the swap metadata stored in the
1631 * OBJT_SWAP object. All swp_*() routines must be called at
1632 * splvm() because swap can be freed up by the low level vm_page
1633 * code which might be called from interrupts beyond what splbio() covers.
1634 *
1635 * Swap metadata is implemented with a global hash and not directly
1636 * linked into the object. Instead the object simply contains
1637 * appropriate tracking counters.
1638 */
1639
1640/*
1641 * SWP_PAGER_HASH() - hash swap meta data
1642 *
1643 * This is an inline helper function which hashes the swapblk given
1644 * the object and page index. It returns a pointer to a pointer
1645 * to the object, or a pointer to a NULL pointer if it could not
1646 * find a swapblk.
1647 *
1648 * This routine must be called at splvm().
1649 */
1650static __inline struct swblock **
1651swp_pager_hash(vm_object_t object, vm_pindex_t index)
1652{
1653 struct swblock **pswap;
1654 struct swblock *swap;
1655
1656 index &= ~SWAP_META_MASK;
1657 pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
1658 while ((swap = *pswap) != NULL) {
1659 if (swap->swb_object == object &&
1660 swap->swb_index == index
1661 ) {
1662 break;
1663 }
1664 pswap = &swap->swb_hnext;
1665 }
1666 return (pswap);
1667}
1668
1669/*
1670 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
1671 *
1672 * We first convert the object to a swap object if it is a default
1673 * object.
1674 *
1675 * The specified swapblk is added to the object's swap metadata. If
1676 * the swapblk is not valid, it is freed instead. Any previously
1677 * assigned swapblk is freed.
1678 *
1679 * This routine must be called at splvm(), except when used to convert
1680 * an OBJT_DEFAULT object into an OBJT_SWAP object.
1681 */
1682static void
1683swp_pager_meta_build(
1684 vm_object_t object,
1685 vm_pindex_t index,
1686 daddr_t swapblk
1687) {
1688 struct swblock *swap;
1689 struct swblock **pswap;
1690
1691 GIANT_REQUIRED;
1692 /*
1693 * Convert default object to swap object if necessary
1694 */
1695 if (object->type != OBJT_SWAP) {
1696 object->type = OBJT_SWAP;
1697 object->un_pager.swp.swp_bcount = 0;
1698
1699 mtx_lock(&sw_alloc_mtx);
1700 if (object->handle != NULL) {
1701 TAILQ_INSERT_TAIL(
1702 NOBJLIST(object->handle),
1703 object,
1704 pager_object_list
1705 );
1706 } else {
1707 TAILQ_INSERT_TAIL(
1708 &swap_pager_un_object_list,
1709 object,
1710 pager_object_list
1711 );
1712 }
1713 mtx_unlock(&sw_alloc_mtx);
1714 }
1715
1716 /*
1717 * Locate hash entry. If not found create, but if we aren't adding
1718 * anything just return. If we run out of space in the map we wait
1719 * and, since the hash table may have changed, retry.
1720 */
1721retry:
1722 pswap = swp_pager_hash(object, index);
1723
1724 if ((swap = *pswap) == NULL) {
1725 int i;
1726
1727 if (swapblk == SWAPBLK_NONE)
1728 return;
1729
1730 swap = *pswap = zalloc(swap_zone);
1731 if (swap == NULL) {
1732 VM_WAIT;
1733 goto retry;
1734 }
1735 swap->swb_hnext = NULL;
1736 swap->swb_object = object;
1737 swap->swb_index = index & ~SWAP_META_MASK;
1738 swap->swb_count = 0;
1739
1740 ++object->un_pager.swp.swp_bcount;
1741
1742 for (i = 0; i < SWAP_META_PAGES; ++i)
1743 swap->swb_pages[i] = SWAPBLK_NONE;
1744 }
1745
1746 /*
1747 * Delete prior contents of metadata
1748 */
1749 index &= SWAP_META_MASK;
1750
1751 if (swap->swb_pages[index] != SWAPBLK_NONE) {
1752 swp_pager_freeswapspace(swap->swb_pages[index], 1);
1753 --swap->swb_count;
1754 }
1755
1756 /*
1757 * Enter block into metadata
1758 */
1759 swap->swb_pages[index] = swapblk;
1760 if (swapblk != SWAPBLK_NONE)
1761 ++swap->swb_count;
1762}
1763
1764/*
1765 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
1766 *
1767 * The requested range of blocks is freed, with any associated swap
1768 * returned to the swap bitmap.
1769 *
1770 * This routine will free swap metadata structures as they are cleaned
1771 * out. This routine does *NOT* operate on swap metadata associated
1772 * with resident pages.
1773 *
1774 * This routine must be called at splvm()
1775 */
1776static void
1777swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
1778{
1779 GIANT_REQUIRED;
1780
1781 if (object->type != OBJT_SWAP)
1782 return;
1783
1784 while (count > 0) {
1785 struct swblock **pswap;
1786 struct swblock *swap;
1787
1788 pswap = swp_pager_hash(object, index);
1789
1790 if ((swap = *pswap) != NULL) {
1791 daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
1792
1793 if (v != SWAPBLK_NONE) {
1794 swp_pager_freeswapspace(v, 1);
1795 swap->swb_pages[index & SWAP_META_MASK] =
1796 SWAPBLK_NONE;
1797 if (--swap->swb_count == 0) {
1798 *pswap = swap->swb_hnext;
1799 zfree(swap_zone, swap);
1800 --object->un_pager.swp.swp_bcount;
1801 }
1802 }
1803 --count;
1804 ++index;
1805 } else {
1806 int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
1807 count -= n;
1808 index += n;
1809 }
1810 }
1811}
1812
1813/*
1814 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
1815 *
1816 * This routine locates and destroys all swap metadata associated with
1817 * an object.
1818 *
1819 * This routine must be called at splvm()
1820 */
1821static void
1822swp_pager_meta_free_all(vm_object_t object)
1823{
1824 daddr_t index = 0;
1825
1826 GIANT_REQUIRED;
1827
1828 if (object->type != OBJT_SWAP)
1829 return;
1830
1831 while (object->un_pager.swp.swp_bcount) {
1832 struct swblock **pswap;
1833 struct swblock *swap;
1834
1835 pswap = swp_pager_hash(object, index);
1836 if ((swap = *pswap) != NULL) {
1837 int i;
1838
1839 for (i = 0; i < SWAP_META_PAGES; ++i) {
1840 daddr_t v = swap->swb_pages[i];
1841 if (v != SWAPBLK_NONE) {
1842 --swap->swb_count;
1843 swp_pager_freeswapspace(v, 1);
1844 }
1845 }
1846 if (swap->swb_count != 0)
1847 panic("swap_pager_meta_free_all: swb_count != 0");
1848 *pswap = swap->swb_hnext;
1849 zfree(swap_zone, swap);
1850 --object->un_pager.swp.swp_bcount;
1851 }
1852 index += SWAP_META_PAGES;
1853 if (index > 0x20000000)
1854 panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
1855 }
1856}
1857
1858/*
1859 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data.
1860 *
1861 * This routine is capable of looking up, popping, or freeing
1862 * swapblk assignments in the swap meta data or in the vm_page_t.
1863 * The routine typically returns the swapblk being looked-up, or popped,
1864 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
1865 * was invalid. This routine will automatically free any invalid
1866 * meta-data swapblks.
1867 *
1868 * It is not possible to store invalid swapblks in the swap meta data
1869 * (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
1870 *
1871 * When acting on a busy resident page and paging is in progress, we
1872 * have to wait until paging is complete but otherwise can act on the
1873 * busy page.
1874 *
1875 * This routine must be called at splvm().
1876 *
1877 * SWM_FREE remove and free swap block from metadata
1878 * SWM_POP remove from meta data but do not free.. pop it out
1879 */
1880static daddr_t
1881swp_pager_meta_ctl(
1882 vm_object_t object,
1883 vm_pindex_t index,
1884 int flags
1885) {
1886 struct swblock **pswap;
1887 struct swblock *swap;
1888 daddr_t r1;
1889
1890 GIANT_REQUIRED;
1891 /*
1892 * The meta data only exists of the object is OBJT_SWAP
1893 * and even then might not be allocated yet.
1894 */
1895 if (object->type != OBJT_SWAP)
1896 return (SWAPBLK_NONE);
1897
1898 r1 = SWAPBLK_NONE;
1899 pswap = swp_pager_hash(object, index);
1900
1901 if ((swap = *pswap) != NULL) {
1902 index &= SWAP_META_MASK;
1903 r1 = swap->swb_pages[index];
1904
1905 if (r1 != SWAPBLK_NONE) {
1906 if (flags & SWM_FREE) {
1907 swp_pager_freeswapspace(r1, 1);
1908 r1 = SWAPBLK_NONE;
1909 }
1910 if (flags & (SWM_FREE|SWM_POP)) {
1911 swap->swb_pages[index] = SWAPBLK_NONE;
1912 if (--swap->swb_count == 0) {
1913 *pswap = swap->swb_hnext;
1914 zfree(swap_zone, swap);
1915 --object->un_pager.swp.swp_bcount;
1916 }
1917 }
1918 }
1919 }
1920 return (r1);
1921}
1922
1923/********************************************************
1924 * CHAINING FUNCTIONS *
1925 ********************************************************
1926 *
1927 * These functions support recursion of I/O operations
1928 * on bp's, typically by chaining one or more 'child' bp's
1929 * to the parent. Synchronous, asynchronous, and semi-synchronous
1930 * chaining is possible.
1931 */
1932
1933/*
1934 * vm_pager_chain_iodone:
1935 *
1936 * io completion routine for child bp. Currently we fudge a bit
1937 * on dealing with b_resid. Since users of these routines may issue
1938 * multiple children simultaneously, sequencing of the error can be lost.
1939 */
1940static void
1941vm_pager_chain_iodone(struct buf *nbp)
1942{
1943 struct bio *bp;
1944 u_int *count;
1945
1946 bp = nbp->b_caller1;
1947 count = (u_int *)&(bp->bio_driver1);
1948 if (bp != NULL) {
1949 if (nbp->b_ioflags & BIO_ERROR) {
1950 bp->bio_flags |= BIO_ERROR;
1951 bp->bio_error = nbp->b_error;
1952 } else if (nbp->b_resid != 0) {
1953 bp->bio_flags |= BIO_ERROR;
1954 bp->bio_error = EINVAL;
1955 } else {
1956 bp->bio_resid -= nbp->b_bcount;
1957 }
1958 nbp->b_caller1 = NULL;
1959 --(*count);
1960 if (bp->bio_flags & BIO_FLAG1) {
1961 bp->bio_flags &= ~BIO_FLAG1;
1962 wakeup(bp);
1963 }
1964 }
1965 nbp->b_flags |= B_DONE;
1966 nbp->b_flags &= ~B_ASYNC;
1967 relpbuf(nbp, NULL);
1968}
1969
1970/*
1971 * getchainbuf:
1972 *
1973 * Obtain a physical buffer and chain it to its parent buffer. When
1974 * I/O completes, the parent buffer will be B_SIGNAL'd. Errors are
1975 * automatically propagated to the parent
1976 */
1977struct buf *
1978getchainbuf(struct bio *bp, struct vnode *vp, int flags)
1979{
1980 struct buf *nbp;
1981 u_int *count;
1982
1983 GIANT_REQUIRED;
1984 nbp = getpbuf(NULL);
1985 count = (u_int *)&(bp->bio_driver1);
1986
1987 nbp->b_caller1 = bp;
1988 ++(*count);
1989
1990 if (*count > 4)
1991 waitchainbuf(bp, 4, 0);
1992
1993 nbp->b_iocmd = bp->bio_cmd;
1994 nbp->b_ioflags = 0;
1995 nbp->b_flags = flags;
1996 nbp->b_rcred = crhold(thread0.td_ucred);
1997 nbp->b_wcred = crhold(thread0.td_ucred);
1998 nbp->b_iodone = vm_pager_chain_iodone;
1999
2000 if (vp)
2001 pbgetvp(vp, nbp);
2002 return (nbp);
2003}
2004
2005void
2006flushchainbuf(struct buf *nbp)
2007{
2008 GIANT_REQUIRED;
2009 if (nbp->b_bcount) {
2010 nbp->b_bufsize = nbp->b_bcount;
2011 if (nbp->b_iocmd == BIO_WRITE)
2012 nbp->b_dirtyend = nbp->b_bcount;
2013 BUF_KERNPROC(nbp);
2014 BUF_STRATEGY(nbp);
2015 } else {
2016 bufdone(nbp);
2017 }
2018}
2019
2020static void
2021waitchainbuf(struct bio *bp, int limit, int done)
2022{
2023 int s;
2024 u_int *count;
2025
2026 GIANT_REQUIRED;
2027 count = (u_int *)&(bp->bio_driver1);
2028 s = splbio();
2029 while (*count > limit) {
2030 bp->bio_flags |= BIO_FLAG1;
2031 tsleep(bp, PRIBIO + 4, "bpchain", 0);
2032 }
2033 if (done) {
2034 if (bp->bio_resid != 0 && !(bp->bio_flags & BIO_ERROR)) {
2035 bp->bio_flags |= BIO_ERROR;
2036 bp->bio_error = EINVAL;
2037 }
2038 biodone(bp);
2039 }
2040 splx(s);
2041}
2042
|