swap_pager.c (118286) | swap_pager.c (118390) |
---|---|
1/* 2 * Copyright (c) 1998 Matthew Dillon, 3 * Copyright (c) 1994 John S. Dyson 4 * Copyright (c) 1990 University of Utah. 5 * Copyright (c) 1982, 1986, 1989, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by --- 53 unchanged lines hidden (view full) --- 62 * 63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 64 * 65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 67 */ 68 69#include <sys/cdefs.h> | 1/* 2 * Copyright (c) 1998 Matthew Dillon, 3 * Copyright (c) 1994 John S. Dyson 4 * Copyright (c) 1990 University of Utah. 5 * Copyright (c) 1982, 1986, 1989, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by --- 53 unchanged lines hidden (view full) --- 62 * 63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 64 * 65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 67 */ 68 69#include <sys/cdefs.h> |
70__FBSDID("$FreeBSD: head/sys/vm/swap_pager.c 118286 2003-07-31 22:19:28Z phk $"); | 70__FBSDID("$FreeBSD: head/sys/vm/swap_pager.c 118390 2003-08-03 13:35:31Z phk $"); |
71 72#include "opt_mac.h" 73#include "opt_swap.h" 74#include "opt_vm.h" 75 76#include <sys/param.h> 77#include <sys/systm.h> 78#include <sys/conf.h> --- 23 unchanged lines hidden (view full) --- 102#include <vm/vm_page.h> 103#include <vm/vm_pager.h> 104#include <vm/vm_pageout.h> 105#include <vm/vm_param.h> 106#include <vm/swap_pager.h> 107#include <vm/vm_extern.h> 108#include <vm/uma.h> 109 | 71 72#include "opt_mac.h" 73#include "opt_swap.h" 74#include "opt_vm.h" 75 76#include <sys/param.h> 77#include <sys/systm.h> 78#include <sys/conf.h> --- 23 unchanged lines hidden (view full) --- 102#include <vm/vm_page.h> 103#include <vm/vm_pager.h> 104#include <vm/vm_pageout.h> 105#include <vm/vm_param.h> 106#include <vm/swap_pager.h> 107#include <vm/vm_extern.h> 108#include <vm/uma.h> 109 |
110#ifndef NSWAPDEV 111#define NSWAPDEV 4 112#endif 113 | |
114/* 115 * SWB_NPAGES must be a power of 2. It may be set to 1, 2, 4, 8, or 16 116 * pages per allocation. We recommend you stick with the default of 8. 117 * The 16-page limit is due to the radix code (kern/subr_blist.c). 118 */ 119#ifndef MAX_PAGEOUT_CLUSTER 120#define MAX_PAGEOUT_CLUSTER 16 121#endif --- 11 unchanged lines hidden (view full) --- 133 * 2 MBytes. This is acceptable. 134 * 135 * Overall memory utilization is about the same as the old swap structure. 136 */ 137#define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t)) 138#define SWAP_META_PAGES (SWB_NPAGES * 2) 139#define SWAP_META_MASK (SWAP_META_PAGES - 1) 140 | 110/* 111 * SWB_NPAGES must be a power of 2. It may be set to 1, 2, 4, 8, or 16 112 * pages per allocation. We recommend you stick with the default of 8. 113 * The 16-page limit is due to the radix code (kern/subr_blist.c). 114 */ 115#ifndef MAX_PAGEOUT_CLUSTER 116#define MAX_PAGEOUT_CLUSTER 16 117#endif --- 11 unchanged lines hidden (view full) --- 129 * 2 MBytes. This is acceptable. 130 * 131 * Overall memory utilization is about the same as the old swap structure. 132 */ 133#define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t)) 134#define SWAP_META_PAGES (SWB_NPAGES * 2) 135#define SWAP_META_MASK (SWAP_META_PAGES - 1) 136 |
141typedef int32_t swblk_t; /* swap offset */ | 137typedef int32_t swblk_t; /* 138 * swap offset. This is the type used to 139 * address the "virtual swap device" and 140 * therefore the maximum swap space is 141 * 2^32 pages. 142 */ |
142 143/* 144 * Swap device table 145 */ 146struct swdevt { 147 udev_t sw_dev; /* For quasibogus swapdev reporting */ 148 int sw_flags; 149 int sw_nblks; 150 int sw_used; 151 struct vnode *sw_vp; 152 dev_t sw_device; | 143 144/* 145 * Swap device table 146 */ 147struct swdevt { 148 udev_t sw_dev; /* For quasibogus swapdev reporting */ 149 int sw_flags; 150 int sw_nblks; 151 int sw_used; 152 struct vnode *sw_vp; 153 dev_t sw_device; |
154 swblk_t sw_first; 155 swblk_t sw_end; 156 struct blist *sw_blist; 157 TAILQ_ENTRY(swdevt) sw_list; |
|
153}; 154 155#define SW_CLOSING 0x04 156 157struct swblock { 158 struct swblock *swb_hnext; 159 vm_object_t swb_object; 160 vm_pindex_t swb_index; 161 int swb_count; 162 daddr_t swb_pages[SWAP_META_PAGES]; 163}; 164 | 158}; 159 160#define SW_CLOSING 0x04 161 162struct swblock { 163 struct swblock *swb_hnext; 164 vm_object_t swb_object; 165 vm_pindex_t swb_index; 166 int swb_count; 167 daddr_t swb_pages[SWAP_META_PAGES]; 168}; 169 |
165static struct swdevt swdevt[NSWAPDEV]; 166static int nswap; /* first block after the interleaved devs */ 167int vm_swap_size; | 170static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq); 171static struct swdevt *swdevhd; /* Allocate from here next */ 172static int nswapdev; /* Number of swap devices */ 173int swap_pager_avail; |
168static int swdev_syscall_active = 0; /* serialize swap(on|off) */ 169 170static int swapdev_strategy(struct vop_strategy_args *ap); 171static struct vnode *swapdev_vp; 172 173#define SWM_FREE 0x02 /* free, period */ 174#define SWM_POP 0x04 /* pop out */ 175 176int swap_pager_full; /* swap space exhaustion (task killing) */ 177static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/ 178static int nsw_rcount; /* free read buffers */ 179static int nsw_wcount_sync; /* limit write buffers / synchronous */ 180static int nsw_wcount_async; /* limit write buffers / asynchronous */ 181static int nsw_wcount_async_max;/* assigned maximum */ 182static int nsw_cluster_max; /* maximum VOP I/O allowed */ 183 | 174static int swdev_syscall_active = 0; /* serialize swap(on|off) */ 175 176static int swapdev_strategy(struct vop_strategy_args *ap); 177static struct vnode *swapdev_vp; 178 179#define SWM_FREE 0x02 /* free, period */ 180#define SWM_POP 0x04 /* pop out */ 181 182int swap_pager_full; /* swap space exhaustion (task killing) */ 183static int swap_pager_almost_full; /* swap space exhaustion (w/ hysteresis)*/ 184static int nsw_rcount; /* free read buffers */ 185static int nsw_wcount_sync; /* limit write buffers / synchronous */ 186static int nsw_wcount_async; /* limit write buffers / asynchronous */ 187static int nsw_wcount_async_max;/* assigned maximum */ 188static int nsw_cluster_max; /* maximum VOP I/O allowed */ 189 |
184static struct blist *swapblist; | |
185static struct swblock **swhash; 186static int swhash_mask; 187static int swap_async_max = 4; /* maximum in-progress async I/O's */ 188static struct sx sw_alloc_sx; 189 190 191SYSCTL_INT(_vm, OID_AUTO, swap_async_max, 192 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops"); 193 | 190static struct swblock **swhash; 191static int swhash_mask; 192static int swap_async_max = 4; /* maximum in-progress async I/O's */ 193static struct sx sw_alloc_sx; 194 195 196SYSCTL_INT(_vm, OID_AUTO, swap_async_max, 197 CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops"); 198 |
194#define BLK2DEVIDX(blk) (NSWAPDEV > 1 ? blk / dmmax % NSWAPDEV : 0) 195 | |
196/* 197 * "named" and "unnamed" anon region objects. Try to reduce the overhead 198 * of searching a named list by hashing it just a little. 199 */ 200 201#define NOBJLISTS 8 202 203#define NOBJLIST(handle) \ --- 14 unchanged lines hidden (view full) --- 218 vm_prot_t prot, vm_ooffset_t offset); 219static void swap_pager_dealloc(vm_object_t object); 220static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int); 221static boolean_t 222 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); 223static void swap_pager_init(void); 224static void swap_pager_unswapped(vm_page_t); 225static void swap_pager_strategy(vm_object_t, struct bio *); | 199/* 200 * "named" and "unnamed" anon region objects. Try to reduce the overhead 201 * of searching a named list by hashing it just a little. 202 */ 203 204#define NOBJLISTS 8 205 206#define NOBJLIST(handle) \ --- 14 unchanged lines hidden (view full) --- 221 vm_prot_t prot, vm_ooffset_t offset); 222static void swap_pager_dealloc(vm_object_t object); 223static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int); 224static boolean_t 225 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); 226static void swap_pager_init(void); 227static void swap_pager_unswapped(vm_page_t); 228static void swap_pager_strategy(vm_object_t, struct bio *); |
226static void swap_pager_swapoff(int devidx, int *sw_used); | 229static void swap_pager_swapoff(struct swdevt *sp, int *sw_used); |
227 228struct pagerops swappagerops = { 229 swap_pager_init, /* early system initialization of pager */ 230 swap_pager_alloc, /* allocate an OBJT_SWAP object */ 231 swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ 232 swap_pager_getpages, /* pagein */ 233 swap_pager_putpages, /* pageout */ 234 swap_pager_haspage, /* get backing store status for page */ --- 50 unchanged lines hidden (view full) --- 285 * This routine may not block. 286 * This routine must be called at splvm() 287 */ 288static void 289swp_sizecheck() 290{ 291 GIANT_REQUIRED; 292 | 230 231struct pagerops swappagerops = { 232 swap_pager_init, /* early system initialization of pager */ 233 swap_pager_alloc, /* allocate an OBJT_SWAP object */ 234 swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ 235 swap_pager_getpages, /* pagein */ 236 swap_pager_putpages, /* pageout */ 237 swap_pager_haspage, /* get backing store status for page */ --- 50 unchanged lines hidden (view full) --- 288 * This routine may not block. 289 * This routine must be called at splvm() 290 */ 291static void 292swp_sizecheck() 293{ 294 GIANT_REQUIRED; 295 |
293 if (vm_swap_size < nswap_lowat) { | 296 if (swap_pager_avail < nswap_lowat) { |
294 if (swap_pager_almost_full == 0) { 295 printf("swap_pager: out of swap space\n"); 296 swap_pager_almost_full = 1; 297 } 298 } else { 299 swap_pager_full = 0; | 297 if (swap_pager_almost_full == 0) { 298 printf("swap_pager: out of swap space\n"); 299 swap_pager_almost_full = 1; 300 } 301 } else { 302 swap_pager_full = 0; |
300 if (vm_swap_size > nswap_hiwat) | 303 if (swap_pager_avail > nswap_hiwat) |
301 swap_pager_almost_full = 0; 302 } 303} 304 305/* 306 * SWP_PAGER_HASH() - hash swap meta data 307 * 308 * This is an helper function which hashes the swapblk given --- 247 unchanged lines hidden (view full) --- 556 * Also has the side effect of advising that somebody made a mistake 557 * when they configured swap and didn't configure enough. 558 * 559 * Must be called at splvm() to avoid races with bitmap frees from 560 * vm_page_remove() aka swap_pager_page_removed(). 561 * 562 * This routine may not block 563 * This routine must be called at splvm(). | 304 swap_pager_almost_full = 0; 305 } 306} 307 308/* 309 * SWP_PAGER_HASH() - hash swap meta data 310 * 311 * This is an helper function which hashes the swapblk given --- 247 unchanged lines hidden (view full) --- 559 * Also has the side effect of advising that somebody made a mistake 560 * when they configured swap and didn't configure enough. 561 * 562 * Must be called at splvm() to avoid races with bitmap frees from 563 * vm_page_remove() aka swap_pager_page_removed(). 564 * 565 * This routine may not block 566 * This routine must be called at splvm(). |
567 * 568 * We allocate in round-robin fashion from the configured devices. |
|
564 */ 565static daddr_t 566swp_pager_getswapspace(npages) 567 int npages; 568{ 569 daddr_t blk; | 569 */ 570static daddr_t 571swp_pager_getswapspace(npages) 572 int npages; 573{ 574 daddr_t blk; |
575 struct swdevt *sp; 576 int i; |
|
570 571 GIANT_REQUIRED; 572 | 577 578 GIANT_REQUIRED; 579 |
573 if ((blk = blist_alloc(swapblist, npages)) == SWAPBLK_NONE) { 574 if (swap_pager_full != 2) { 575 printf("swap_pager_getswapspace: failed\n"); 576 swap_pager_full = 2; 577 swap_pager_almost_full = 1; | 580 blk = SWAPBLK_NONE; 581 sp = swdevhd; 582 for (i = 0; i < nswapdev; i++) { 583 if (sp == NULL) 584 sp = TAILQ_FIRST(&swtailq); 585 if (!(sp->sw_flags & SW_CLOSING)) { 586 blk = blist_alloc(sp->sw_blist, npages); 587 if (blk != SWAPBLK_NONE) { 588 blk += sp->sw_first; 589 swap_pager_avail -= npages; 590 sp->sw_used += npages; 591 swp_sizecheck(); 592 swdevhd = TAILQ_NEXT(sp, sw_list); 593 return(blk); 594 } |
578 } | 595 } |
579 } else { 580 vm_swap_size -= npages; 581 /* per-swap area stats */ 582 swdevt[BLK2DEVIDX(blk)].sw_used += npages; 583 swp_sizecheck(); | 596 sp = TAILQ_NEXT(sp, sw_list); |
584 } | 597 } |
598 if (swap_pager_full != 2) { 599 printf("swap_pager_getswapspace: failed\n"); 600 swap_pager_full = 2; 601 swap_pager_almost_full = 1; 602 } 603 swdevhd = NULL; |
|
585 return (blk); 586} 587 | 604 return (blk); 605} 606 |
607static struct swdevt * 608swap_pager_find_dev(daddr_t blk, int npages) 609{ 610 struct swdevt *sp; 611 612 TAILQ_FOREACH(sp, &swtailq, sw_list) { 613 if (blk >= sp->sw_first && blk + npages <= sp->sw_end) 614 return (sp); 615 } 616 printf("Failed to find swapdev blk %ju, %d pages\n", 617 (uintmax_t)blk, npages); 618 TAILQ_FOREACH(sp, &swtailq, sw_list) 619 printf("has %ju...%ju\n", 620 (uintmax_t)sp->sw_first, (uintmax_t)sp->sw_end); 621 return (NULL); 622} 623 624 |
|
588/* 589 * SWP_PAGER_FREESWAPSPACE() - free raw swap space 590 * 591 * This routine returns the specified swap blocks back to the bitmap. 592 * 593 * Note: This routine may not block (it could in the old swap code), 594 * and through the use of the new blist routines it does not block. 595 * 596 * We must be called at splvm() to avoid races with bitmap frees from 597 * vm_page_remove() aka swap_pager_page_removed(). 598 * 599 * This routine may not block 600 * This routine must be called at splvm(). 601 */ 602static void | 625/* 626 * SWP_PAGER_FREESWAPSPACE() - free raw swap space 627 * 628 * This routine returns the specified swap blocks back to the bitmap. 629 * 630 * Note: This routine may not block (it could in the old swap code), 631 * and through the use of the new blist routines it does not block. 632 * 633 * We must be called at splvm() to avoid races with bitmap frees from 634 * vm_page_remove() aka swap_pager_page_removed(). 635 * 636 * This routine may not block 637 * This routine must be called at splvm(). 638 */ 639static void |
603swp_pager_freeswapspace(blk, npages) 604 daddr_t blk; 605 int npages; | 640swp_pager_freeswapspace(daddr_t blk, int npages) |
606{ | 641{ |
607 struct swdevt *sp = &swdevt[BLK2DEVIDX(blk)]; | 642 struct swdevt *sp; |
608 609 GIANT_REQUIRED; 610 | 643 644 GIANT_REQUIRED; 645 |
646 sp = swap_pager_find_dev(blk, npages); 647 |
|
611 /* per-swap area stats */ 612 sp->sw_used -= npages; 613 614 /* 615 * If we are attempting to stop swapping on this device, we 616 * don't want to mark any blocks free lest they be reused. 617 */ 618 if (sp->sw_flags & SW_CLOSING) 619 return; 620 | 648 /* per-swap area stats */ 649 sp->sw_used -= npages; 650 651 /* 652 * If we are attempting to stop swapping on this device, we 653 * don't want to mark any blocks free lest they be reused. 654 */ 655 if (sp->sw_flags & SW_CLOSING) 656 return; 657 |
621 blist_free(swapblist, blk, npages); 622 vm_swap_size += npages; | 658 blist_free(sp->sw_blist, blk - sp->sw_first, npages); 659 swap_pager_avail += npages; |
623 swp_sizecheck(); 624} 625 626/* 627 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page 628 * range within an object. 629 * 630 * This is a globally accessible routine. --- 1111 unchanged lines hidden (view full) --- 1742/* 1743 * swap_pager_isswapped: 1744 * 1745 * Return 1 if at least one page in the given object is paged 1746 * out to the given swap device. 1747 * 1748 * This routine may not block. 1749 */ | 660 swp_sizecheck(); 661} 662 663/* 664 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page 665 * range within an object. 666 * 667 * This is a globally accessible routine. --- 1111 unchanged lines hidden (view full) --- 1779/* 1780 * swap_pager_isswapped: 1781 * 1782 * Return 1 if at least one page in the given object is paged 1783 * out to the given swap device. 1784 * 1785 * This routine may not block. 1786 */ |
1750int swap_pager_isswapped(vm_object_t object, int devidx) { | 1787int 1788swap_pager_isswapped(vm_object_t object, struct swdevt *sp) 1789{ |
1751 daddr_t index = 0; 1752 int bcount; 1753 int i; 1754 1755 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1756 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) { 1757 struct swblock *swap; 1758 1759 if ((swap = *swp_pager_hash(object, index)) != NULL) { 1760 for (i = 0; i < SWAP_META_PAGES; ++i) { 1761 daddr_t v = swap->swb_pages[i]; | 1790 daddr_t index = 0; 1791 int bcount; 1792 int i; 1793 1794 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1795 for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) { 1796 struct swblock *swap; 1797 1798 if ((swap = *swp_pager_hash(object, index)) != NULL) { 1799 for (i = 0; i < SWAP_META_PAGES; ++i) { 1800 daddr_t v = swap->swb_pages[i]; |
1762 if (v != SWAPBLK_NONE && 1763 BLK2DEVIDX(v) == devidx) | 1801 if (v == SWAPBLK_NONE) 1802 continue; 1803 if (swap_pager_find_dev(v, 1) == sp) |
1764 return 1; 1765 } 1766 } 1767 1768 index += SWAP_META_PAGES; 1769 if (index > 0x20000000) 1770 panic("swap_pager_isswapped: failed to locate all swap meta blocks"); 1771 } --- 65 unchanged lines hidden (view full) --- 1837 * The sw_used parameter points to the field in the swdev structure 1838 * that contains a count of the number of blocks still allocated 1839 * on the device. If we encounter objects with a nonzero pip count 1840 * in our scan, we use this number to determine if we're really done. 1841 * 1842 * This routine may block. 1843 */ 1844static void | 1804 return 1; 1805 } 1806 } 1807 1808 index += SWAP_META_PAGES; 1809 if (index > 0x20000000) 1810 panic("swap_pager_isswapped: failed to locate all swap meta blocks"); 1811 } --- 65 unchanged lines hidden (view full) --- 1877 * The sw_used parameter points to the field in the swdev structure 1878 * that contains a count of the number of blocks still allocated 1879 * on the device. If we encounter objects with a nonzero pip count 1880 * in our scan, we use this number to determine if we're really done. 1881 * 1882 * This routine may block. 1883 */ 1884static void |
1845swap_pager_swapoff(int devidx, int *sw_used) | 1885swap_pager_swapoff(struct swdevt *sp, int *sw_used) |
1846{ 1847 struct swblock **pswap; 1848 struct swblock *swap; 1849 vm_object_t waitobj; 1850 daddr_t v; 1851 int i, j; 1852 1853 GIANT_REQUIRED; 1854 1855full_rescan: 1856 waitobj = NULL; 1857 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */ 1858restart: 1859 pswap = &swhash[i]; 1860 while ((swap = *pswap) != NULL) { 1861 for (j = 0; j < SWAP_META_PAGES; ++j) { 1862 v = swap->swb_pages[j]; 1863 if (v != SWAPBLK_NONE && | 1886{ 1887 struct swblock **pswap; 1888 struct swblock *swap; 1889 vm_object_t waitobj; 1890 daddr_t v; 1891 int i, j; 1892 1893 GIANT_REQUIRED; 1894 1895full_rescan: 1896 waitobj = NULL; 1897 for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */ 1898restart: 1899 pswap = &swhash[i]; 1900 while ((swap = *pswap) != NULL) { 1901 for (j = 0; j < SWAP_META_PAGES; ++j) { 1902 v = swap->swb_pages[j]; 1903 if (v != SWAPBLK_NONE && |
1864 BLK2DEVIDX(v) == devidx) | 1904 swap_pager_find_dev(v, 1) == sp) |
1865 break; 1866 } 1867 if (j < SWAP_META_PAGES) { 1868 swp_pager_force_pagein(swap, j); 1869 goto restart; 1870 } else if (swap->swb_object->paging_in_progress) { 1871 if (!waitobj) 1872 waitobj = swap->swb_object; --- 416 unchanged lines hidden (view full) --- 2289 */ 2290static int 2291swapdev_strategy(ap) 2292 struct vop_strategy_args /* { 2293 struct vnode *a_vp; 2294 struct buf *a_bp; 2295 } */ *ap; 2296{ | 1905 break; 1906 } 1907 if (j < SWAP_META_PAGES) { 1908 swp_pager_force_pagein(swap, j); 1909 goto restart; 1910 } else if (swap->swb_object->paging_in_progress) { 1911 if (!waitobj) 1912 waitobj = swap->swb_object; --- 416 unchanged lines hidden (view full) --- 2329 */ 2330static int 2331swapdev_strategy(ap) 2332 struct vop_strategy_args /* { 2333 struct vnode *a_vp; 2334 struct buf *a_bp; 2335 } */ *ap; 2336{ |
2297 int s, sz, off, seg, index; | 2337 int s, sz; |
2298 struct swdevt *sp; 2299 struct vnode *vp; 2300 struct buf *bp; 2301 2302 KASSERT(ap->a_vp == ap->a_bp->b_vp, ("%s(%p != %p)", 2303 __func__, ap->a_vp, ap->a_bp->b_vp)); 2304 bp = ap->a_bp; 2305 sz = howmany(bp->b_bcount, PAGE_SIZE); 2306 2307 /* 2308 * Convert interleaved swap into per-device swap. Note that 2309 * the block size is left in PAGE_SIZE'd chunks (for the newswap) 2310 * here. 2311 */ | 2338 struct swdevt *sp; 2339 struct vnode *vp; 2340 struct buf *bp; 2341 2342 KASSERT(ap->a_vp == ap->a_bp->b_vp, ("%s(%p != %p)", 2343 __func__, ap->a_vp, ap->a_bp->b_vp)); 2344 bp = ap->a_bp; 2345 sz = howmany(bp->b_bcount, PAGE_SIZE); 2346 2347 /* 2348 * Convert interleaved swap into per-device swap. Note that 2349 * the block size is left in PAGE_SIZE'd chunks (for the newswap) 2350 * here. 2351 */ |
2312 if (NSWAPDEV > 1) { 2313 off = bp->b_blkno % dmmax; 2314 if (off + sz > dmmax) { 2315 bp->b_error = EINVAL; 2316 bp->b_ioflags |= BIO_ERROR; 2317 bufdone(bp); 2318 return 0; 2319 } 2320 seg = bp->b_blkno / dmmax; 2321 index = seg % NSWAPDEV; 2322 seg /= NSWAPDEV; 2323 bp->b_blkno = seg * dmmax + off; 2324 } else { 2325 index = 0; 2326 } 2327 sp = &swdevt[index]; 2328 if (bp->b_blkno + sz > sp->sw_nblks) { 2329 bp->b_error = EINVAL; 2330 bp->b_ioflags |= BIO_ERROR; 2331 bufdone(bp); 2332 return 0; 2333 } | 2352 sp = swap_pager_find_dev(bp->b_blkno, sz); |
2334 bp->b_dev = sp->sw_device; | 2353 bp->b_dev = sp->sw_device; |
2335 if (sp->sw_vp == NULL) { 2336 bp->b_error = ENODEV; 2337 bp->b_ioflags |= BIO_ERROR; 2338 bufdone(bp); 2339 return 0; 2340 } 2341 | |
2342 /* 2343 * Convert from PAGE_SIZE'd to DEV_BSIZE'd chunks for the actual I/O 2344 */ | 2354 /* 2355 * Convert from PAGE_SIZE'd to DEV_BSIZE'd chunks for the actual I/O 2356 */ |
2345 bp->b_blkno = ctodb(bp->b_blkno); | 2357 bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first); |
2346 2347 vhold(sp->sw_vp); 2348 s = splvm(); 2349 if (bp->b_iocmd == BIO_WRITE) { 2350 vp = bp->b_vp; 2351 if (vp) { 2352 VI_LOCK(vp); 2353 vp->v_numoutput--; --- 116 unchanged lines hidden (view full) --- 2470 */ 2471int 2472swaponvp(td, vp, dev, nblks) 2473 struct thread *td; 2474 struct vnode *vp; 2475 dev_t dev; 2476 u_long nblks; 2477{ | 2358 2359 vhold(sp->sw_vp); 2360 s = splvm(); 2361 if (bp->b_iocmd == BIO_WRITE) { 2362 vp = bp->b_vp; 2363 if (vp) { 2364 VI_LOCK(vp); 2365 vp->v_numoutput--; --- 116 unchanged lines hidden (view full) --- 2482 */ 2483int 2484swaponvp(td, vp, dev, nblks) 2485 struct thread *td; 2486 struct vnode *vp; 2487 dev_t dev; 2488 u_long nblks; 2489{ |
2478 int index; | |
2479 struct swdevt *sp; | 2490 struct swdevt *sp; |
2480 swblk_t vsbase; 2481 long blk; | |
2482 swblk_t dvbase; 2483 int error; | 2491 swblk_t dvbase; 2492 int error; |
2484 u_long aligned_nblks, mblocks; | 2493 u_long mblocks; |
2485 off_t mediasize; 2486 2487 if (!swapdev_vp) { 2488 error = getnewvnode("none", NULL, swapdev_vnodeop_p, 2489 &swapdev_vp); 2490 if (error) 2491 panic("Cannot get vnode for swapdev"); 2492 swapdev_vp->v_type = VNON; /* Untyped */ 2493 } 2494 2495 ASSERT_VOP_UNLOCKED(vp, "swaponvp"); | 2494 off_t mediasize; 2495 2496 if (!swapdev_vp) { 2497 error = getnewvnode("none", NULL, swapdev_vnodeop_p, 2498 &swapdev_vp); 2499 if (error) 2500 panic("Cannot get vnode for swapdev"); 2501 swapdev_vp->v_type = VNON; /* Untyped */ 2502 } 2503 2504 ASSERT_VOP_UNLOCKED(vp, "swaponvp"); |
2496 for (sp = swdevt, index = 0 ; index < NSWAPDEV; index++, sp++) { | 2505 dvbase = 0; 2506 TAILQ_FOREACH(sp, &swtailq, sw_list) { |
2497 if (sp->sw_vp == vp) | 2507 if (sp->sw_vp == vp) |
2498 return EBUSY; 2499 if (!sp->sw_vp) 2500 goto found; 2501 | 2508 return (EBUSY); 2509 if (sp->sw_end >= dvbase) 2510 dvbase = sp->sw_end; |
2502 } | 2511 } |
2503 return EINVAL; 2504 found: | 2512 |
2505 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); 2506#ifdef MAC 2507 error = mac_check_system_swapon(td->td_ucred, vp); 2508 if (error == 0) 2509#endif 2510 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1); 2511 (void) VOP_UNLOCK(vp, 0, td); 2512 if (error) --- 13 unchanged lines hidden (view full) --- 2526 (void) VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td); 2527 return (ENXIO); 2528 } 2529 2530 /* 2531 * If we go beyond this, we get overflows in the radix 2532 * tree bitmap code. 2533 */ | 2513 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); 2514#ifdef MAC 2515 error = mac_check_system_swapon(td->td_ucred, vp); 2516 if (error == 0) 2517#endif 2518 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1); 2519 (void) VOP_UNLOCK(vp, 0, td); 2520 if (error) --- 13 unchanged lines hidden (view full) --- 2534 (void) VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td); 2535 return (ENXIO); 2536 } 2537 2538 /* 2539 * If we go beyond this, we get overflows in the radix 2540 * tree bitmap code. 2541 */ |
2534 mblocks = 0x40000000 / BLIST_META_RADIX / NSWAPDEV; | 2542 mblocks = 0x40000000 / BLIST_META_RADIX; |
2535 if (nblks > mblocks) { 2536 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n", 2537 mblocks); 2538 nblks = mblocks; 2539 } 2540 /* 2541 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. 2542 * First chop nblks off to page-align it, then convert. 2543 * 2544 * sw->sw_nblks is in page-sized chunks now too. 2545 */ 2546 nblks &= ~(ctodb(1) - 1); 2547 nblks = dbtoc(nblks); 2548 | 2543 if (nblks > mblocks) { 2544 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n", 2545 mblocks); 2546 nblks = mblocks; 2547 } 2548 /* 2549 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. 2550 * First chop nblks off to page-align it, then convert. 2551 * 2552 * sw->sw_nblks is in page-sized chunks now too. 2553 */ 2554 nblks &= ~(ctodb(1) - 1); 2555 nblks = dbtoc(nblks); 2556 |
2557 sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO); |
|
2549 sp->sw_vp = vp; 2550 sp->sw_dev = dev2udev(dev); 2551 sp->sw_device = dev; 2552 sp->sw_flags = 0; 2553 sp->sw_nblks = nblks; 2554 sp->sw_used = 0; | 2558 sp->sw_vp = vp; 2559 sp->sw_dev = dev2udev(dev); 2560 sp->sw_device = dev; 2561 sp->sw_flags = 0; 2562 sp->sw_nblks = nblks; 2563 sp->sw_used = 0; |
2564 sp->sw_first = dvbase; 2565 sp->sw_end = dvbase + nblks; |
|
2555 | 2566 |
2567 sp->sw_blist = blist_create(nblks); |
|
2556 /* | 2568 /* |
2557 * nblks, nswap, and dmmax are PAGE_SIZE'd parameters now, not 2558 * DEV_BSIZE'd. aligned_nblks is used to calculate the 2559 * size of the swap bitmap, taking into account the stripe size. | 2569 * Do not free the first block in order to avoid overwriting 2570 * any bsd label at the front of the partition |
2560 */ | 2571 */ |
2561 aligned_nblks = (nblks + (dmmax -1)) & ~(u_long)(dmmax -1); | 2572 blist_free(sp->sw_blist, 1, nblks); |
2562 | 2573 |
2563 if (aligned_nblks * NSWAPDEV > nswap) 2564 nswap = aligned_nblks * NSWAPDEV; 2565 2566 if (swapblist == NULL) 2567 swapblist = blist_create(nswap); 2568 else 2569 blist_resize(&swapblist, nswap, 0); 2570 2571 for (dvbase = dmmax; dvbase < nblks; dvbase += dmmax) { 2572 blk = min(nblks - dvbase, dmmax); 2573 vsbase = index * dmmax + dvbase * NSWAPDEV; 2574 blist_free(swapblist, vsbase, blk); 2575 vm_swap_size += blk; 2576 } 2577 | 2574 TAILQ_INSERT_TAIL(&swtailq, sp, sw_list); 2575 nswapdev++; 2576 swap_pager_avail += nblks; |
2578 swap_pager_full = 0; 2579 2580 return (0); 2581} 2582 2583/* 2584 * SYSCALL: swapoff(devname) 2585 * --- 12 unchanged lines hidden (view full) --- 2598int 2599swapoff(td, uap) 2600 struct thread *td; 2601 struct swapoff_args *uap; 2602{ 2603 struct vnode *vp; 2604 struct nameidata nd; 2605 struct swdevt *sp; | 2577 swap_pager_full = 0; 2578 2579 return (0); 2580} 2581 2582/* 2583 * SYSCALL: swapoff(devname) 2584 * --- 12 unchanged lines hidden (view full) --- 2597int 2598swapoff(td, uap) 2599 struct thread *td; 2600 struct swapoff_args *uap; 2601{ 2602 struct vnode *vp; 2603 struct nameidata nd; 2604 struct swdevt *sp; |
2606 swblk_t dvbase, vsbase; 2607 u_long nblks, aligned_nblks, blk; 2608 int error, index; | 2605 u_long nblks, dvbase; 2606 int error; |
2609 2610 mtx_lock(&Giant); 2611 2612 error = suser(td); 2613 if (error) 2614 goto done2; 2615 2616 while (swdev_syscall_active) 2617 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0); 2618 swdev_syscall_active = 1; 2619 2620 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td); 2621 error = namei(&nd); 2622 if (error) 2623 goto done; 2624 NDFREE(&nd, NDF_ONLY_PNBUF); 2625 vp = nd.ni_vp; 2626 | 2607 2608 mtx_lock(&Giant); 2609 2610 error = suser(td); 2611 if (error) 2612 goto done2; 2613 2614 while (swdev_syscall_active) 2615 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0); 2616 swdev_syscall_active = 1; 2617 2618 NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td); 2619 error = namei(&nd); 2620 if (error) 2621 goto done; 2622 NDFREE(&nd, NDF_ONLY_PNBUF); 2623 vp = nd.ni_vp; 2624 |
2627 for (sp = swdevt, index = 0 ; index < NSWAPDEV; index++, sp++) { | 2625 TAILQ_FOREACH(sp, &swtailq, sw_list) { |
2628 if (sp->sw_vp == vp) 2629 goto found; 2630 } 2631 error = EINVAL; 2632 goto done; 2633found: 2634#ifdef MAC 2635 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); --- 6 unchanged lines hidden (view full) --- 2642 nblks = sp->sw_nblks; 2643 2644 /* 2645 * We can turn off this swap device safely only if the 2646 * available virtual memory in the system will fit the amount 2647 * of data we will have to page back in, plus an epsilon so 2648 * the system doesn't become critically low on swap space. 2649 */ | 2626 if (sp->sw_vp == vp) 2627 goto found; 2628 } 2629 error = EINVAL; 2630 goto done; 2631found: 2632#ifdef MAC 2633 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); --- 6 unchanged lines hidden (view full) --- 2640 nblks = sp->sw_nblks; 2641 2642 /* 2643 * We can turn off this swap device safely only if the 2644 * available virtual memory in the system will fit the amount 2645 * of data we will have to page back in, plus an epsilon so 2646 * the system doesn't become critically low on swap space. 2647 */ |
2650 if (cnt.v_free_count + cnt.v_cache_count + vm_swap_size < | 2648 if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail < |
2651 nblks + nswap_lowat) { 2652 error = ENOMEM; 2653 goto done; 2654 } 2655 2656 /* 2657 * Prevent further allocations on this device. 2658 */ 2659 sp->sw_flags |= SW_CLOSING; | 2649 nblks + nswap_lowat) { 2650 error = ENOMEM; 2651 goto done; 2652 } 2653 2654 /* 2655 * Prevent further allocations on this device. 2656 */ 2657 sp->sw_flags |= SW_CLOSING; |
2660 for (dvbase = dmmax; dvbase < nblks; dvbase += dmmax) { 2661 blk = min(nblks - dvbase, dmmax); 2662 vsbase = index * dmmax + dvbase * NSWAPDEV; 2663 vm_swap_size -= blist_fill(swapblist, vsbase, blk); | 2658 for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) { 2659 swap_pager_avail -= blist_fill(sp->sw_blist, 2660 dvbase, dmmax); |
2664 } 2665 2666 /* 2667 * Page in the contents of the device and close it. 2668 */ 2669#ifndef NO_SWAPPING | 2661 } 2662 2663 /* 2664 * Page in the contents of the device and close it. 2665 */ 2666#ifndef NO_SWAPPING |
2670 vm_proc_swapin_all(index); | 2667 vm_proc_swapin_all(sp); |
2671#endif /* !NO_SWAPPING */ | 2668#endif /* !NO_SWAPPING */ |
2672 swap_pager_swapoff(index, &sp->sw_used); | 2669 swap_pager_swapoff(sp, &sp->sw_used); |
2673 2674 VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td); 2675 vrele(vp); 2676 sp->sw_vp = NULL; | 2670 2671 VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td); 2672 vrele(vp); 2673 sp->sw_vp = NULL; |
2674 TAILQ_REMOVE(&swtailq, sp, sw_list); 2675 if (swdevhd == sp) 2676 swdevhd = NULL; 2677 nswapdev--; 2678 blist_destroy(sp->sw_blist); 2679 free(sp, M_VMPGDATA); |
|
2677 | 2680 |
2678 /* 2679 * Resize the bitmap based on the new largest swap device, 2680 * or free the bitmap if there are no more devices. 2681 */ 2682 for (sp = swdevt, nblks = 0; sp < swdevt + NSWAPDEV; sp++) { 2683 if (sp->sw_vp == NULL) 2684 continue; 2685 nblks = max(nblks, sp->sw_nblks); 2686 } 2687 2688 aligned_nblks = (nblks + (dmmax -1)) & ~(u_long)(dmmax -1); 2689 nswap = aligned_nblks * NSWAPDEV; 2690 2691 if (nswap == 0) { 2692 blist_destroy(swapblist); 2693 swapblist = NULL; 2694 vrele(swapdev_vp); 2695 swapdev_vp = NULL; 2696 } else 2697 blist_resize(&swapblist, nswap, 0); 2698 | |
2699done: 2700 swdev_syscall_active = 0; 2701 wakeup_one(&swdev_syscall_active); 2702done2: 2703 mtx_unlock(&Giant); 2704 return (error); 2705} 2706 2707void 2708swap_pager_status(int *total, int *used) 2709{ 2710 struct swdevt *sp; | 2681done: 2682 swdev_syscall_active = 0; 2683 wakeup_one(&swdev_syscall_active); 2684done2: 2685 mtx_unlock(&Giant); 2686 return (error); 2687} 2688 2689void 2690swap_pager_status(int *total, int *used) 2691{ 2692 struct swdevt *sp; |
2711 int i; | |
2712 2713 *total = 0; 2714 *used = 0; | 2693 2694 *total = 0; 2695 *used = 0; |
2715 for (sp = swdevt, i = 0; i < NSWAPDEV; i++, sp++) { 2716 if (sp->sw_vp == NULL) 2717 continue; | 2696 TAILQ_FOREACH(sp, &swtailq, sw_list) { |
2718 *total += sp->sw_nblks; 2719 *used += sp->sw_used; 2720 } 2721} 2722 2723static int 2724sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS) 2725{ 2726 int *name = (int *)arg1; | 2697 *total += sp->sw_nblks; 2698 *used += sp->sw_used; 2699 } 2700} 2701 2702static int 2703sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS) 2704{ 2705 int *name = (int *)arg1; |
2727 int error, i, n; | 2706 int error, n; |
2728 struct xswdev xs; 2729 struct swdevt *sp; 2730 2731 if (arg2 != 1) /* name length */ 2732 return (EINVAL); 2733 | 2707 struct xswdev xs; 2708 struct swdevt *sp; 2709 2710 if (arg2 != 1) /* name length */ 2711 return (EINVAL); 2712 |
2734 for (sp = swdevt, i = 0, n = 0 ; i < NSWAPDEV; i++, sp++) { 2735 if (sp->sw_vp) { 2736 if (n == *name) { 2737 xs.xsw_version = XSWDEV_VERSION; 2738 xs.xsw_dev = sp->sw_dev; 2739 xs.xsw_flags = sp->sw_flags; 2740 xs.xsw_nblks = sp->sw_nblks; 2741 xs.xsw_used = sp->sw_used; | 2713 n = 0; 2714 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2715 if (n == *name) { 2716 xs.xsw_version = XSWDEV_VERSION; 2717 xs.xsw_dev = sp->sw_dev; 2718 xs.xsw_flags = sp->sw_flags; 2719 xs.xsw_nblks = sp->sw_nblks; 2720 xs.xsw_used = sp->sw_used; |
2742 | 2721 |
2743 error = SYSCTL_OUT(req, &xs, sizeof(xs)); 2744 return (error); 2745 } 2746 n++; | 2722 error = SYSCTL_OUT(req, &xs, sizeof(xs)); 2723 return (error); |
2747 } | 2724 } |
2748 | 2725 n++; |
2749 } 2750 return (ENOENT); 2751} 2752 | 2726 } 2727 return (ENOENT); 2728} 2729 |
2753SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, 0, NSWAPDEV, | 2730SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0, |
2754 "Number of swap devices"); 2755SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info, 2756 "Swap statistics by device"); 2757 2758/* 2759 * vmspace_swap_count() - count the approximate swap useage in pages for a 2760 * vmspace. 2761 * --- 24 unchanged lines hidden (view full) --- 2786 count += object->un_pager.swp.swp_bcount * 2787 SWAP_META_PAGES * n / object->size + 1; 2788 } 2789 VM_OBJECT_UNLOCK(object); 2790 } 2791 } 2792 return (count); 2793} | 2731 "Number of swap devices"); 2732SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info, 2733 "Swap statistics by device"); 2734 2735/* 2736 * vmspace_swap_count() - count the approximate swap useage in pages for a 2737 * vmspace. 2738 * --- 24 unchanged lines hidden (view full) --- 2763 count += object->un_pager.swp.swp_bcount * 2764 SWAP_META_PAGES * n / object->size + 1; 2765 } 2766 VM_OBJECT_UNLOCK(object); 2767 } 2768 } 2769 return (count); 2770} |
2794 | |