Deleted Added
sdiff udiff text old ( 118286 ) new ( 118390 )
full compact
swap_pager.c (118286) swap_pager.c (118390)
1/*
2 * Copyright (c) 1998 Matthew Dillon,
3 * Copyright (c) 1994 John S. Dyson
4 * Copyright (c) 1990 University of Utah.
5 * Copyright (c) 1982, 1986, 1989, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by

--- 53 unchanged lines hidden (view full) ---

62 *
63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64 *
65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94
66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94
67 */
68
69#include <sys/cdefs.h>
1/*
2 * Copyright (c) 1998 Matthew Dillon,
3 * Copyright (c) 1994 John S. Dyson
4 * Copyright (c) 1990 University of Utah.
5 * Copyright (c) 1982, 1986, 1989, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by

--- 53 unchanged lines hidden (view full) ---

62 *
63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
64 *
65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94
66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94
67 */
68
69#include <sys/cdefs.h>
70__FBSDID("$FreeBSD: head/sys/vm/swap_pager.c 118286 2003-07-31 22:19:28Z phk $");
70__FBSDID("$FreeBSD: head/sys/vm/swap_pager.c 118390 2003-08-03 13:35:31Z phk $");
71
72#include "opt_mac.h"
73#include "opt_swap.h"
74#include "opt_vm.h"
75
76#include <sys/param.h>
77#include <sys/systm.h>
78#include <sys/conf.h>

--- 23 unchanged lines hidden (view full) ---

102#include <vm/vm_page.h>
103#include <vm/vm_pager.h>
104#include <vm/vm_pageout.h>
105#include <vm/vm_param.h>
106#include <vm/swap_pager.h>
107#include <vm/vm_extern.h>
108#include <vm/uma.h>
109
71
72#include "opt_mac.h"
73#include "opt_swap.h"
74#include "opt_vm.h"
75
76#include <sys/param.h>
77#include <sys/systm.h>
78#include <sys/conf.h>

--- 23 unchanged lines hidden (view full) ---

102#include <vm/vm_page.h>
103#include <vm/vm_pager.h>
104#include <vm/vm_pageout.h>
105#include <vm/vm_param.h>
106#include <vm/swap_pager.h>
107#include <vm/vm_extern.h>
108#include <vm/uma.h>
109
110#ifndef NSWAPDEV
111#define NSWAPDEV 4
112#endif
113
114/*
115 * SWB_NPAGES must be a power of 2. It may be set to 1, 2, 4, 8, or 16
116 * pages per allocation. We recommend you stick with the default of 8.
117 * The 16-page limit is due to the radix code (kern/subr_blist.c).
118 */
119#ifndef MAX_PAGEOUT_CLUSTER
120#define MAX_PAGEOUT_CLUSTER 16
121#endif

--- 11 unchanged lines hidden (view full) ---

133 * 2 MBytes. This is acceptable.
134 *
135 * Overall memory utilization is about the same as the old swap structure.
136 */
137#define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
138#define SWAP_META_PAGES (SWB_NPAGES * 2)
139#define SWAP_META_MASK (SWAP_META_PAGES - 1)
140
110/*
111 * SWB_NPAGES must be a power of 2. It may be set to 1, 2, 4, 8, or 16
112 * pages per allocation. We recommend you stick with the default of 8.
113 * The 16-page limit is due to the radix code (kern/subr_blist.c).
114 */
115#ifndef MAX_PAGEOUT_CLUSTER
116#define MAX_PAGEOUT_CLUSTER 16
117#endif

--- 11 unchanged lines hidden (view full) ---

129 * 2 MBytes. This is acceptable.
130 *
131 * Overall memory utilization is about the same as the old swap structure.
132 */
133#define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
134#define SWAP_META_PAGES (SWB_NPAGES * 2)
135#define SWAP_META_MASK (SWAP_META_PAGES - 1)
136
141typedef int32_t swblk_t; /* swap offset */
137typedef int32_t swblk_t; /*
138 * swap offset. This is the type used to
139 * address the "virtual swap device" and
140 * therefore the maximum swap space is
141 * 2^32 pages.
142 */
142
143/*
144 * Swap device table
145 */
146struct swdevt {
147 udev_t sw_dev; /* For quasibogus swapdev reporting */
148 int sw_flags;
149 int sw_nblks;
150 int sw_used;
151 struct vnode *sw_vp;
152 dev_t sw_device;
143
144/*
145 * Swap device table
146 */
147struct swdevt {
148 udev_t sw_dev; /* For quasibogus swapdev reporting */
149 int sw_flags;
150 int sw_nblks;
151 int sw_used;
152 struct vnode *sw_vp;
153 dev_t sw_device;
154 swblk_t sw_first;
155 swblk_t sw_end;
156 struct blist *sw_blist;
157 TAILQ_ENTRY(swdevt) sw_list;
153};
154
155#define SW_CLOSING 0x04
156
157struct swblock {
158 struct swblock *swb_hnext;
159 vm_object_t swb_object;
160 vm_pindex_t swb_index;
161 int swb_count;
162 daddr_t swb_pages[SWAP_META_PAGES];
163};
164
158};
159
160#define SW_CLOSING 0x04
161
162struct swblock {
163 struct swblock *swb_hnext;
164 vm_object_t swb_object;
165 vm_pindex_t swb_index;
166 int swb_count;
167 daddr_t swb_pages[SWAP_META_PAGES];
168};
169
165static struct swdevt swdevt[NSWAPDEV];
166static int nswap; /* first block after the interleaved devs */
167int vm_swap_size;
170static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
171static struct swdevt *swdevhd; /* Allocate from here next */
172static int nswapdev; /* Number of swap devices */
173int swap_pager_avail;
168static int swdev_syscall_active = 0; /* serialize swap(on|off) */
169
170static int swapdev_strategy(struct vop_strategy_args *ap);
171static struct vnode *swapdev_vp;
172
173#define SWM_FREE 0x02 /* free, period */
174#define SWM_POP 0x04 /* pop out */
175
176int swap_pager_full; /* swap space exhaustion (task killing) */
177static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/
178static int nsw_rcount; /* free read buffers */
179static int nsw_wcount_sync; /* limit write buffers / synchronous */
180static int nsw_wcount_async; /* limit write buffers / asynchronous */
181static int nsw_wcount_async_max;/* assigned maximum */
182static int nsw_cluster_max; /* maximum VOP I/O allowed */
183
174static int swdev_syscall_active = 0; /* serialize swap(on|off) */
175
176static int swapdev_strategy(struct vop_strategy_args *ap);
177static struct vnode *swapdev_vp;
178
179#define SWM_FREE 0x02 /* free, period */
180#define SWM_POP 0x04 /* pop out */
181
182int swap_pager_full; /* swap space exhaustion (task killing) */
183static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/
184static int nsw_rcount; /* free read buffers */
185static int nsw_wcount_sync; /* limit write buffers / synchronous */
186static int nsw_wcount_async; /* limit write buffers / asynchronous */
187static int nsw_wcount_async_max;/* assigned maximum */
188static int nsw_cluster_max; /* maximum VOP I/O allowed */
189
184static struct blist *swapblist;
185static struct swblock **swhash;
186static int swhash_mask;
187static int swap_async_max = 4; /* maximum in-progress async I/O's */
188static struct sx sw_alloc_sx;
189
190
191SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
192 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
193
190static struct swblock **swhash;
191static int swhash_mask;
192static int swap_async_max = 4; /* maximum in-progress async I/O's */
193static struct sx sw_alloc_sx;
194
195
196SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
197 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
198
194#define BLK2DEVIDX(blk) (NSWAPDEV > 1 ? blk / dmmax % NSWAPDEV : 0)
195
196/*
197 * "named" and "unnamed" anon region objects. Try to reduce the overhead
198 * of searching a named list by hashing it just a little.
199 */
200
201#define NOBJLISTS 8
202
203#define NOBJLIST(handle) \

--- 14 unchanged lines hidden (view full) ---

218 vm_prot_t prot, vm_ooffset_t offset);
219static void swap_pager_dealloc(vm_object_t object);
220static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
221static boolean_t
222 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
223static void swap_pager_init(void);
224static void swap_pager_unswapped(vm_page_t);
225static void swap_pager_strategy(vm_object_t, struct bio *);
199/*
200 * "named" and "unnamed" anon region objects. Try to reduce the overhead
201 * of searching a named list by hashing it just a little.
202 */
203
204#define NOBJLISTS 8
205
206#define NOBJLIST(handle) \

--- 14 unchanged lines hidden (view full) ---

221 vm_prot_t prot, vm_ooffset_t offset);
222static void swap_pager_dealloc(vm_object_t object);
223static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
224static boolean_t
225 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
226static void swap_pager_init(void);
227static void swap_pager_unswapped(vm_page_t);
228static void swap_pager_strategy(vm_object_t, struct bio *);
226static void swap_pager_swapoff(int devidx, int *sw_used);
229static void swap_pager_swapoff(struct swdevt *sp, int *sw_used);
227
228struct pagerops swappagerops = {
229 swap_pager_init, /* early system initialization of pager */
230 swap_pager_alloc, /* allocate an OBJT_SWAP object */
231 swap_pager_dealloc, /* deallocate an OBJT_SWAP object */
232 swap_pager_getpages, /* pagein */
233 swap_pager_putpages, /* pageout */
234 swap_pager_haspage, /* get backing store status for page */

--- 50 unchanged lines hidden (view full) ---

285 * This routine may not block.
286 * This routine must be called at splvm()
287 */
288static void
289swp_sizecheck()
290{
291 GIANT_REQUIRED;
292
230
231struct pagerops swappagerops = {
232 swap_pager_init, /* early system initialization of pager */
233 swap_pager_alloc, /* allocate an OBJT_SWAP object */
234 swap_pager_dealloc, /* deallocate an OBJT_SWAP object */
235 swap_pager_getpages, /* pagein */
236 swap_pager_putpages, /* pageout */
237 swap_pager_haspage, /* get backing store status for page */

--- 50 unchanged lines hidden (view full) ---

288 * This routine may not block.
289 * This routine must be called at splvm()
290 */
291static void
292swp_sizecheck()
293{
294 GIANT_REQUIRED;
295
293 if (vm_swap_size < nswap_lowat) {
296 if (swap_pager_avail < nswap_lowat) {
294 if (swap_pager_almost_full == 0) {
295 printf("swap_pager: out of swap space\n");
296 swap_pager_almost_full = 1;
297 }
298 } else {
299 swap_pager_full = 0;
297 if (swap_pager_almost_full == 0) {
298 printf("swap_pager: out of swap space\n");
299 swap_pager_almost_full = 1;
300 }
301 } else {
302 swap_pager_full = 0;
300 if (vm_swap_size > nswap_hiwat)
303 if (swap_pager_avail > nswap_hiwat)
301 swap_pager_almost_full = 0;
302 }
303}
304
305/*
306 * SWP_PAGER_HASH() - hash swap meta data
307 *
308 * This is an helper function which hashes the swapblk given

--- 247 unchanged lines hidden (view full) ---

556 * Also has the side effect of advising that somebody made a mistake
557 * when they configured swap and didn't configure enough.
558 *
559 * Must be called at splvm() to avoid races with bitmap frees from
560 * vm_page_remove() aka swap_pager_page_removed().
561 *
562 * This routine may not block
563 * This routine must be called at splvm().
304 swap_pager_almost_full = 0;
305 }
306}
307
308/*
309 * SWP_PAGER_HASH() - hash swap meta data
310 *
311 * This is an helper function which hashes the swapblk given

--- 247 unchanged lines hidden (view full) ---

559 * Also has the side effect of advising that somebody made a mistake
560 * when they configured swap and didn't configure enough.
561 *
562 * Must be called at splvm() to avoid races with bitmap frees from
563 * vm_page_remove() aka swap_pager_page_removed().
564 *
565 * This routine may not block
566 * This routine must be called at splvm().
567 *
568 * We allocate in round-robin fashion from the configured devices.
564 */
565static daddr_t
566swp_pager_getswapspace(npages)
567 int npages;
568{
569 daddr_t blk;
569 */
570static daddr_t
571swp_pager_getswapspace(npages)
572 int npages;
573{
574 daddr_t blk;
575 struct swdevt *sp;
576 int i;
570
571 GIANT_REQUIRED;
572
577
578 GIANT_REQUIRED;
579
573 if ((blk = blist_alloc(swapblist, npages)) == SWAPBLK_NONE) {
574 if (swap_pager_full != 2) {
575 printf("swap_pager_getswapspace: failed\n");
576 swap_pager_full = 2;
577 swap_pager_almost_full = 1;
580 blk = SWAPBLK_NONE;
581 sp = swdevhd;
582 for (i = 0; i < nswapdev; i++) {
583 if (sp == NULL)
584 sp = TAILQ_FIRST(&swtailq);
585 if (!(sp->sw_flags & SW_CLOSING)) {
586 blk = blist_alloc(sp->sw_blist, npages);
587 if (blk != SWAPBLK_NONE) {
588 blk += sp->sw_first;
589 swap_pager_avail -= npages;
590 sp->sw_used += npages;
591 swp_sizecheck();
592 swdevhd = TAILQ_NEXT(sp, sw_list);
593 return(blk);
594 }
578 }
595 }
579 } else {
580 vm_swap_size -= npages;
581 /* per-swap area stats */
582 swdevt[BLK2DEVIDX(blk)].sw_used += npages;
583 swp_sizecheck();
596 sp = TAILQ_NEXT(sp, sw_list);
584 }
597 }
598 if (swap_pager_full != 2) {
599 printf("swap_pager_getswapspace: failed\n");
600 swap_pager_full = 2;
601 swap_pager_almost_full = 1;
602 }
603 swdevhd = NULL;
585 return (blk);
586}
587
604 return (blk);
605}
606
607static struct swdevt *
608swap_pager_find_dev(daddr_t blk, int npages)
609{
610 struct swdevt *sp;
611
612 TAILQ_FOREACH(sp, &swtailq, sw_list) {
613 if (blk >= sp->sw_first && blk + npages <= sp->sw_end)
614 return (sp);
615 }
616 printf("Failed to find swapdev blk %ju, %d pages\n",
617 (uintmax_t)blk, npages);
618 TAILQ_FOREACH(sp, &swtailq, sw_list)
619 printf("has %ju...%ju\n",
620 (uintmax_t)sp->sw_first, (uintmax_t)sp->sw_end);
621 return (NULL);
622}
623
624
588/*
589 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
590 *
591 * This routine returns the specified swap blocks back to the bitmap.
592 *
593 * Note: This routine may not block (it could in the old swap code),
594 * and through the use of the new blist routines it does not block.
595 *
596 * We must be called at splvm() to avoid races with bitmap frees from
597 * vm_page_remove() aka swap_pager_page_removed().
598 *
599 * This routine may not block
600 * This routine must be called at splvm().
601 */
602static void
625/*
626 * SWP_PAGER_FREESWAPSPACE() - free raw swap space
627 *
628 * This routine returns the specified swap blocks back to the bitmap.
629 *
630 * Note: This routine may not block (it could in the old swap code),
631 * and through the use of the new blist routines it does not block.
632 *
633 * We must be called at splvm() to avoid races with bitmap frees from
634 * vm_page_remove() aka swap_pager_page_removed().
635 *
636 * This routine may not block
637 * This routine must be called at splvm().
638 */
639static void
603swp_pager_freeswapspace(blk, npages)
604 daddr_t blk;
605 int npages;
640swp_pager_freeswapspace(daddr_t blk, int npages)
606{
641{
607 struct swdevt *sp = &swdevt[BLK2DEVIDX(blk)];
642 struct swdevt *sp;
608
609 GIANT_REQUIRED;
610
643
644 GIANT_REQUIRED;
645
646 sp = swap_pager_find_dev(blk, npages);
647
611 /* per-swap area stats */
612 sp->sw_used -= npages;
613
614 /*
615 * If we are attempting to stop swapping on this device, we
616 * don't want to mark any blocks free lest they be reused.
617 */
618 if (sp->sw_flags & SW_CLOSING)
619 return;
620
648 /* per-swap area stats */
649 sp->sw_used -= npages;
650
651 /*
652 * If we are attempting to stop swapping on this device, we
653 * don't want to mark any blocks free lest they be reused.
654 */
655 if (sp->sw_flags & SW_CLOSING)
656 return;
657
621 blist_free(swapblist, blk, npages);
622 vm_swap_size += npages;
658 blist_free(sp->sw_blist, blk - sp->sw_first, npages);
659 swap_pager_avail += npages;
623 swp_sizecheck();
624}
625
626/*
627 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
628 * range within an object.
629 *
630 * This is a globally accessible routine.

--- 1111 unchanged lines hidden (view full) ---

1742/*
1743 * swap_pager_isswapped:
1744 *
1745 * Return 1 if at least one page in the given object is paged
1746 * out to the given swap device.
1747 *
1748 * This routine may not block.
1749 */
660 swp_sizecheck();
661}
662
663/*
664 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page
665 * range within an object.
666 *
667 * This is a globally accessible routine.

--- 1111 unchanged lines hidden (view full) ---

1779/*
1780 * swap_pager_isswapped:
1781 *
1782 * Return 1 if at least one page in the given object is paged
1783 * out to the given swap device.
1784 *
1785 * This routine may not block.
1786 */
1750int swap_pager_isswapped(vm_object_t object, int devidx) {
1787int
1788swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
1789{
1751 daddr_t index = 0;
1752 int bcount;
1753 int i;
1754
1755 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1756 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1757 struct swblock *swap;
1758
1759 if ((swap = *swp_pager_hash(object, index)) != NULL) {
1760 for (i = 0; i < SWAP_META_PAGES; ++i) {
1761 daddr_t v = swap->swb_pages[i];
1790 daddr_t index = 0;
1791 int bcount;
1792 int i;
1793
1794 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1795 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
1796 struct swblock *swap;
1797
1798 if ((swap = *swp_pager_hash(object, index)) != NULL) {
1799 for (i = 0; i < SWAP_META_PAGES; ++i) {
1800 daddr_t v = swap->swb_pages[i];
1762 if (v != SWAPBLK_NONE &&
1763 BLK2DEVIDX(v) == devidx)
1801 if (v == SWAPBLK_NONE)
1802 continue;
1803 if (swap_pager_find_dev(v, 1) == sp)
1764 return 1;
1765 }
1766 }
1767
1768 index += SWAP_META_PAGES;
1769 if (index > 0x20000000)
1770 panic("swap_pager_isswapped: failed to locate all swap meta blocks");
1771 }

--- 65 unchanged lines hidden (view full) ---

1837 * The sw_used parameter points to the field in the swdev structure
1838 * that contains a count of the number of blocks still allocated
1839 * on the device. If we encounter objects with a nonzero pip count
1840 * in our scan, we use this number to determine if we're really done.
1841 *
1842 * This routine may block.
1843 */
1844static void
1804 return 1;
1805 }
1806 }
1807
1808 index += SWAP_META_PAGES;
1809 if (index > 0x20000000)
1810 panic("swap_pager_isswapped: failed to locate all swap meta blocks");
1811 }

--- 65 unchanged lines hidden (view full) ---

1877 * The sw_used parameter points to the field in the swdev structure
1878 * that contains a count of the number of blocks still allocated
1879 * on the device. If we encounter objects with a nonzero pip count
1880 * in our scan, we use this number to determine if we're really done.
1881 *
1882 * This routine may block.
1883 */
1884static void
1845swap_pager_swapoff(int devidx, int *sw_used)
1885swap_pager_swapoff(struct swdevt *sp, int *sw_used)
1846{
1847 struct swblock **pswap;
1848 struct swblock *swap;
1849 vm_object_t waitobj;
1850 daddr_t v;
1851 int i, j;
1852
1853 GIANT_REQUIRED;
1854
1855full_rescan:
1856 waitobj = NULL;
1857 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1858restart:
1859 pswap = &swhash[i];
1860 while ((swap = *pswap) != NULL) {
1861 for (j = 0; j < SWAP_META_PAGES; ++j) {
1862 v = swap->swb_pages[j];
1863 if (v != SWAPBLK_NONE &&
1886{
1887 struct swblock **pswap;
1888 struct swblock *swap;
1889 vm_object_t waitobj;
1890 daddr_t v;
1891 int i, j;
1892
1893 GIANT_REQUIRED;
1894
1895full_rescan:
1896 waitobj = NULL;
1897 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
1898restart:
1899 pswap = &swhash[i];
1900 while ((swap = *pswap) != NULL) {
1901 for (j = 0; j < SWAP_META_PAGES; ++j) {
1902 v = swap->swb_pages[j];
1903 if (v != SWAPBLK_NONE &&
1864 BLK2DEVIDX(v) == devidx)
1904 swap_pager_find_dev(v, 1) == sp)
1865 break;
1866 }
1867 if (j < SWAP_META_PAGES) {
1868 swp_pager_force_pagein(swap, j);
1869 goto restart;
1870 } else if (swap->swb_object->paging_in_progress) {
1871 if (!waitobj)
1872 waitobj = swap->swb_object;

--- 416 unchanged lines hidden (view full) ---

2289 */
2290static int
2291swapdev_strategy(ap)
2292 struct vop_strategy_args /* {
2293 struct vnode *a_vp;
2294 struct buf *a_bp;
2295 } */ *ap;
2296{
1905 break;
1906 }
1907 if (j < SWAP_META_PAGES) {
1908 swp_pager_force_pagein(swap, j);
1909 goto restart;
1910 } else if (swap->swb_object->paging_in_progress) {
1911 if (!waitobj)
1912 waitobj = swap->swb_object;

--- 416 unchanged lines hidden (view full) ---

2329 */
2330static int
2331swapdev_strategy(ap)
2332 struct vop_strategy_args /* {
2333 struct vnode *a_vp;
2334 struct buf *a_bp;
2335 } */ *ap;
2336{
2297 int s, sz, off, seg, index;
2337 int s, sz;
2298 struct swdevt *sp;
2299 struct vnode *vp;
2300 struct buf *bp;
2301
2302 KASSERT(ap->a_vp == ap->a_bp->b_vp, ("%s(%p != %p)",
2303 __func__, ap->a_vp, ap->a_bp->b_vp));
2304 bp = ap->a_bp;
2305 sz = howmany(bp->b_bcount, PAGE_SIZE);
2306
2307 /*
2308 * Convert interleaved swap into per-device swap. Note that
2309 * the block size is left in PAGE_SIZE'd chunks (for the newswap)
2310 * here.
2311 */
2338 struct swdevt *sp;
2339 struct vnode *vp;
2340 struct buf *bp;
2341
2342 KASSERT(ap->a_vp == ap->a_bp->b_vp, ("%s(%p != %p)",
2343 __func__, ap->a_vp, ap->a_bp->b_vp));
2344 bp = ap->a_bp;
2345 sz = howmany(bp->b_bcount, PAGE_SIZE);
2346
2347 /*
2348 * Convert interleaved swap into per-device swap. Note that
2349 * the block size is left in PAGE_SIZE'd chunks (for the newswap)
2350 * here.
2351 */
2312 if (NSWAPDEV > 1) {
2313 off = bp->b_blkno % dmmax;
2314 if (off + sz > dmmax) {
2315 bp->b_error = EINVAL;
2316 bp->b_ioflags |= BIO_ERROR;
2317 bufdone(bp);
2318 return 0;
2319 }
2320 seg = bp->b_blkno / dmmax;
2321 index = seg % NSWAPDEV;
2322 seg /= NSWAPDEV;
2323 bp->b_blkno = seg * dmmax + off;
2324 } else {
2325 index = 0;
2326 }
2327 sp = &swdevt[index];
2328 if (bp->b_blkno + sz > sp->sw_nblks) {
2329 bp->b_error = EINVAL;
2330 bp->b_ioflags |= BIO_ERROR;
2331 bufdone(bp);
2332 return 0;
2333 }
2352 sp = swap_pager_find_dev(bp->b_blkno, sz);
2334 bp->b_dev = sp->sw_device;
2353 bp->b_dev = sp->sw_device;
2335 if (sp->sw_vp == NULL) {
2336 bp->b_error = ENODEV;
2337 bp->b_ioflags |= BIO_ERROR;
2338 bufdone(bp);
2339 return 0;
2340 }
2341
2342 /*
2343 * Convert from PAGE_SIZE'd to DEV_BSIZE'd chunks for the actual I/O
2344 */
2354 /*
2355 * Convert from PAGE_SIZE'd to DEV_BSIZE'd chunks for the actual I/O
2356 */
2345 bp->b_blkno = ctodb(bp->b_blkno);
2357 bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
2346
2347 vhold(sp->sw_vp);
2348 s = splvm();
2349 if (bp->b_iocmd == BIO_WRITE) {
2350 vp = bp->b_vp;
2351 if (vp) {
2352 VI_LOCK(vp);
2353 vp->v_numoutput--;

--- 116 unchanged lines hidden (view full) ---

2470 */
2471int
2472swaponvp(td, vp, dev, nblks)
2473 struct thread *td;
2474 struct vnode *vp;
2475 dev_t dev;
2476 u_long nblks;
2477{
2358
2359 vhold(sp->sw_vp);
2360 s = splvm();
2361 if (bp->b_iocmd == BIO_WRITE) {
2362 vp = bp->b_vp;
2363 if (vp) {
2364 VI_LOCK(vp);
2365 vp->v_numoutput--;

--- 116 unchanged lines hidden (view full) ---

2482 */
2483int
2484swaponvp(td, vp, dev, nblks)
2485 struct thread *td;
2486 struct vnode *vp;
2487 dev_t dev;
2488 u_long nblks;
2489{
2478 int index;
2479 struct swdevt *sp;
2490 struct swdevt *sp;
2480 swblk_t vsbase;
2481 long blk;
2482 swblk_t dvbase;
2483 int error;
2491 swblk_t dvbase;
2492 int error;
2484 u_long aligned_nblks, mblocks;
2493 u_long mblocks;
2485 off_t mediasize;
2486
2487 if (!swapdev_vp) {
2488 error = getnewvnode("none", NULL, swapdev_vnodeop_p,
2489 &swapdev_vp);
2490 if (error)
2491 panic("Cannot get vnode for swapdev");
2492 swapdev_vp->v_type = VNON; /* Untyped */
2493 }
2494
2495 ASSERT_VOP_UNLOCKED(vp, "swaponvp");
2494 off_t mediasize;
2495
2496 if (!swapdev_vp) {
2497 error = getnewvnode("none", NULL, swapdev_vnodeop_p,
2498 &swapdev_vp);
2499 if (error)
2500 panic("Cannot get vnode for swapdev");
2501 swapdev_vp->v_type = VNON; /* Untyped */
2502 }
2503
2504 ASSERT_VOP_UNLOCKED(vp, "swaponvp");
2496 for (sp = swdevt, index = 0 ; index < NSWAPDEV; index++, sp++) {
2505 dvbase = 0;
2506 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2497 if (sp->sw_vp == vp)
2507 if (sp->sw_vp == vp)
2498 return EBUSY;
2499 if (!sp->sw_vp)
2500 goto found;
2501
2508 return (EBUSY);
2509 if (sp->sw_end >= dvbase)
2510 dvbase = sp->sw_end;
2502 }
2511 }
2503 return EINVAL;
2504 found:
2512
2505 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
2506#ifdef MAC
2507 error = mac_check_system_swapon(td->td_ucred, vp);
2508 if (error == 0)
2509#endif
2510 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1);
2511 (void) VOP_UNLOCK(vp, 0, td);
2512 if (error)

--- 13 unchanged lines hidden (view full) ---

2526 (void) VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td);
2527 return (ENXIO);
2528 }
2529
2530 /*
2531 * If we go beyond this, we get overflows in the radix
2532 * tree bitmap code.
2533 */
2513 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
2514#ifdef MAC
2515 error = mac_check_system_swapon(td->td_ucred, vp);
2516 if (error == 0)
2517#endif
2518 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1);
2519 (void) VOP_UNLOCK(vp, 0, td);
2520 if (error)

--- 13 unchanged lines hidden (view full) ---

2534 (void) VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td);
2535 return (ENXIO);
2536 }
2537
2538 /*
2539 * If we go beyond this, we get overflows in the radix
2540 * tree bitmap code.
2541 */
2534 mblocks = 0x40000000 / BLIST_META_RADIX / NSWAPDEV;
2542 mblocks = 0x40000000 / BLIST_META_RADIX;
2535 if (nblks > mblocks) {
2536 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
2537 mblocks);
2538 nblks = mblocks;
2539 }
2540 /*
2541 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2542 * First chop nblks off to page-align it, then convert.
2543 *
2544 * sw->sw_nblks is in page-sized chunks now too.
2545 */
2546 nblks &= ~(ctodb(1) - 1);
2547 nblks = dbtoc(nblks);
2548
2543 if (nblks > mblocks) {
2544 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
2545 mblocks);
2546 nblks = mblocks;
2547 }
2548 /*
2549 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
2550 * First chop nblks off to page-align it, then convert.
2551 *
2552 * sw->sw_nblks is in page-sized chunks now too.
2553 */
2554 nblks &= ~(ctodb(1) - 1);
2555 nblks = dbtoc(nblks);
2556
2557 sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2549 sp->sw_vp = vp;
2550 sp->sw_dev = dev2udev(dev);
2551 sp->sw_device = dev;
2552 sp->sw_flags = 0;
2553 sp->sw_nblks = nblks;
2554 sp->sw_used = 0;
2558 sp->sw_vp = vp;
2559 sp->sw_dev = dev2udev(dev);
2560 sp->sw_device = dev;
2561 sp->sw_flags = 0;
2562 sp->sw_nblks = nblks;
2563 sp->sw_used = 0;
2564 sp->sw_first = dvbase;
2565 sp->sw_end = dvbase + nblks;
2555
2566
2567 sp->sw_blist = blist_create(nblks);
2556 /*
2568 /*
2557 * nblks, nswap, and dmmax are PAGE_SIZE'd parameters now, not
2558 * DEV_BSIZE'd. aligned_nblks is used to calculate the
2559 * size of the swap bitmap, taking into account the stripe size.
2569 * Do not free the first block in order to avoid overwriting
2570 * any bsd label at the front of the partition
2560 */
2571 */
2561 aligned_nblks = (nblks + (dmmax -1)) & ~(u_long)(dmmax -1);
2572 blist_free(sp->sw_blist, 1, nblks);
2562
2573
2563 if (aligned_nblks * NSWAPDEV > nswap)
2564 nswap = aligned_nblks * NSWAPDEV;
2565
2566 if (swapblist == NULL)
2567 swapblist = blist_create(nswap);
2568 else
2569 blist_resize(&swapblist, nswap, 0);
2570
2571 for (dvbase = dmmax; dvbase < nblks; dvbase += dmmax) {
2572 blk = min(nblks - dvbase, dmmax);
2573 vsbase = index * dmmax + dvbase * NSWAPDEV;
2574 blist_free(swapblist, vsbase, blk);
2575 vm_swap_size += blk;
2576 }
2577
2574 TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2575 nswapdev++;
2576 swap_pager_avail += nblks;
2578 swap_pager_full = 0;
2579
2580 return (0);
2581}
2582
2583/*
2584 * SYSCALL: swapoff(devname)
2585 *

--- 12 unchanged lines hidden (view full) ---

2598int
2599swapoff(td, uap)
2600 struct thread *td;
2601 struct swapoff_args *uap;
2602{
2603 struct vnode *vp;
2604 struct nameidata nd;
2605 struct swdevt *sp;
2577 swap_pager_full = 0;
2578
2579 return (0);
2580}
2581
2582/*
2583 * SYSCALL: swapoff(devname)
2584 *

--- 12 unchanged lines hidden (view full) ---

2597int
2598swapoff(td, uap)
2599 struct thread *td;
2600 struct swapoff_args *uap;
2601{
2602 struct vnode *vp;
2603 struct nameidata nd;
2604 struct swdevt *sp;
2606 swblk_t dvbase, vsbase;
2607 u_long nblks, aligned_nblks, blk;
2608 int error, index;
2605 u_long nblks, dvbase;
2606 int error;
2609
2610 mtx_lock(&Giant);
2611
2612 error = suser(td);
2613 if (error)
2614 goto done2;
2615
2616 while (swdev_syscall_active)
2617 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2618 swdev_syscall_active = 1;
2619
2620 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2621 error = namei(&nd);
2622 if (error)
2623 goto done;
2624 NDFREE(&nd, NDF_ONLY_PNBUF);
2625 vp = nd.ni_vp;
2626
2607
2608 mtx_lock(&Giant);
2609
2610 error = suser(td);
2611 if (error)
2612 goto done2;
2613
2614 while (swdev_syscall_active)
2615 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
2616 swdev_syscall_active = 1;
2617
2618 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
2619 error = namei(&nd);
2620 if (error)
2621 goto done;
2622 NDFREE(&nd, NDF_ONLY_PNBUF);
2623 vp = nd.ni_vp;
2624
2627 for (sp = swdevt, index = 0 ; index < NSWAPDEV; index++, sp++) {
2625 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2628 if (sp->sw_vp == vp)
2629 goto found;
2630 }
2631 error = EINVAL;
2632 goto done;
2633found:
2634#ifdef MAC
2635 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);

--- 6 unchanged lines hidden (view full) ---

2642 nblks = sp->sw_nblks;
2643
2644 /*
2645 * We can turn off this swap device safely only if the
2646 * available virtual memory in the system will fit the amount
2647 * of data we will have to page back in, plus an epsilon so
2648 * the system doesn't become critically low on swap space.
2649 */
2626 if (sp->sw_vp == vp)
2627 goto found;
2628 }
2629 error = EINVAL;
2630 goto done;
2631found:
2632#ifdef MAC
2633 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);

--- 6 unchanged lines hidden (view full) ---

2640 nblks = sp->sw_nblks;
2641
2642 /*
2643 * We can turn off this swap device safely only if the
2644 * available virtual memory in the system will fit the amount
2645 * of data we will have to page back in, plus an epsilon so
2646 * the system doesn't become critically low on swap space.
2647 */
2650 if (cnt.v_free_count + cnt.v_cache_count + vm_swap_size <
2648 if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
2651 nblks + nswap_lowat) {
2652 error = ENOMEM;
2653 goto done;
2654 }
2655
2656 /*
2657 * Prevent further allocations on this device.
2658 */
2659 sp->sw_flags |= SW_CLOSING;
2649 nblks + nswap_lowat) {
2650 error = ENOMEM;
2651 goto done;
2652 }
2653
2654 /*
2655 * Prevent further allocations on this device.
2656 */
2657 sp->sw_flags |= SW_CLOSING;
2660 for (dvbase = dmmax; dvbase < nblks; dvbase += dmmax) {
2661 blk = min(nblks - dvbase, dmmax);
2662 vsbase = index * dmmax + dvbase * NSWAPDEV;
2663 vm_swap_size -= blist_fill(swapblist, vsbase, blk);
2658 for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
2659 swap_pager_avail -= blist_fill(sp->sw_blist,
2660 dvbase, dmmax);
2664 }
2665
2666 /*
2667 * Page in the contents of the device and close it.
2668 */
2669#ifndef NO_SWAPPING
2661 }
2662
2663 /*
2664 * Page in the contents of the device and close it.
2665 */
2666#ifndef NO_SWAPPING
2670 vm_proc_swapin_all(index);
2667 vm_proc_swapin_all(sp);
2671#endif /* !NO_SWAPPING */
2668#endif /* !NO_SWAPPING */
2672 swap_pager_swapoff(index, &sp->sw_used);
2669 swap_pager_swapoff(sp, &sp->sw_used);
2673
2674 VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td);
2675 vrele(vp);
2676 sp->sw_vp = NULL;
2670
2671 VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td);
2672 vrele(vp);
2673 sp->sw_vp = NULL;
2674 TAILQ_REMOVE(&swtailq, sp, sw_list);
2675 if (swdevhd == sp)
2676 swdevhd = NULL;
2677 nswapdev--;
2678 blist_destroy(sp->sw_blist);
2679 free(sp, M_VMPGDATA);
2677
2680
2678 /*
2679 * Resize the bitmap based on the new largest swap device,
2680 * or free the bitmap if there are no more devices.
2681 */
2682 for (sp = swdevt, nblks = 0; sp < swdevt + NSWAPDEV; sp++) {
2683 if (sp->sw_vp == NULL)
2684 continue;
2685 nblks = max(nblks, sp->sw_nblks);
2686 }
2687
2688 aligned_nblks = (nblks + (dmmax -1)) & ~(u_long)(dmmax -1);
2689 nswap = aligned_nblks * NSWAPDEV;
2690
2691 if (nswap == 0) {
2692 blist_destroy(swapblist);
2693 swapblist = NULL;
2694 vrele(swapdev_vp);
2695 swapdev_vp = NULL;
2696 } else
2697 blist_resize(&swapblist, nswap, 0);
2698
2699done:
2700 swdev_syscall_active = 0;
2701 wakeup_one(&swdev_syscall_active);
2702done2:
2703 mtx_unlock(&Giant);
2704 return (error);
2705}
2706
2707void
2708swap_pager_status(int *total, int *used)
2709{
2710 struct swdevt *sp;
2681done:
2682 swdev_syscall_active = 0;
2683 wakeup_one(&swdev_syscall_active);
2684done2:
2685 mtx_unlock(&Giant);
2686 return (error);
2687}
2688
2689void
2690swap_pager_status(int *total, int *used)
2691{
2692 struct swdevt *sp;
2711 int i;
2712
2713 *total = 0;
2714 *used = 0;
2693
2694 *total = 0;
2695 *used = 0;
2715 for (sp = swdevt, i = 0; i < NSWAPDEV; i++, sp++) {
2716 if (sp->sw_vp == NULL)
2717 continue;
2696 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2718 *total += sp->sw_nblks;
2719 *used += sp->sw_used;
2720 }
2721}
2722
2723static int
2724sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2725{
2726 int *name = (int *)arg1;
2697 *total += sp->sw_nblks;
2698 *used += sp->sw_used;
2699 }
2700}
2701
2702static int
2703sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
2704{
2705 int *name = (int *)arg1;
2727 int error, i, n;
2706 int error, n;
2728 struct xswdev xs;
2729 struct swdevt *sp;
2730
2731 if (arg2 != 1) /* name length */
2732 return (EINVAL);
2733
2707 struct xswdev xs;
2708 struct swdevt *sp;
2709
2710 if (arg2 != 1) /* name length */
2711 return (EINVAL);
2712
2734 for (sp = swdevt, i = 0, n = 0 ; i < NSWAPDEV; i++, sp++) {
2735 if (sp->sw_vp) {
2736 if (n == *name) {
2737 xs.xsw_version = XSWDEV_VERSION;
2738 xs.xsw_dev = sp->sw_dev;
2739 xs.xsw_flags = sp->sw_flags;
2740 xs.xsw_nblks = sp->sw_nblks;
2741 xs.xsw_used = sp->sw_used;
2713 n = 0;
2714 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2715 if (n == *name) {
2716 xs.xsw_version = XSWDEV_VERSION;
2717 xs.xsw_dev = sp->sw_dev;
2718 xs.xsw_flags = sp->sw_flags;
2719 xs.xsw_nblks = sp->sw_nblks;
2720 xs.xsw_used = sp->sw_used;
2742
2721
2743 error = SYSCTL_OUT(req, &xs, sizeof(xs));
2744 return (error);
2745 }
2746 n++;
2722 error = SYSCTL_OUT(req, &xs, sizeof(xs));
2723 return (error);
2747 }
2724 }
2748
2725 n++;
2749 }
2750 return (ENOENT);
2751}
2752
2726 }
2727 return (ENOENT);
2728}
2729
2753SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, 0, NSWAPDEV,
2730SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
2754 "Number of swap devices");
2755SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2756 "Swap statistics by device");
2757
2758/*
2759 * vmspace_swap_count() - count the approximate swap useage in pages for a
2760 * vmspace.
2761 *

--- 24 unchanged lines hidden (view full) ---

2786 count += object->un_pager.swp.swp_bcount *
2787 SWAP_META_PAGES * n / object->size + 1;
2788 }
2789 VM_OBJECT_UNLOCK(object);
2790 }
2791 }
2792 return (count);
2793}
2731 "Number of swap devices");
2732SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
2733 "Swap statistics by device");
2734
2735/*
2736 * vmspace_swap_count() - count the approximate swap useage in pages for a
2737 * vmspace.
2738 *

--- 24 unchanged lines hidden (view full) ---

2763 count += object->un_pager.swp.swp_bcount *
2764 SWAP_META_PAGES * n / object->size + 1;
2765 }
2766 VM_OBJECT_UNLOCK(object);
2767 }
2768 }
2769 return (count);
2770}
2794