phys_pager.c revision 302408
1214501Srpaulo/*- 2214501Srpaulo * Copyright (c) 2000 Peter Wemm 3214501Srpaulo * 4214501Srpaulo * Redistribution and use in source and binary forms, with or without 5252726Srpaulo * modification, are permitted provided that the following conditions 6252726Srpaulo * are met: 7214501Srpaulo * 1. Redistributions of source code must retain the above copyright 8214501Srpaulo * notice, this list of conditions and the following disclaimer. 9214501Srpaulo * 2. Redistributions in binary form must reproduce the above copyright 10214501Srpaulo * notice, this list of conditions and the following disclaimer in the 11214501Srpaulo * documentation and/or other materials provided with the distribution. 12214501Srpaulo * 13214501Srpaulo * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 14214501Srpaulo * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15214501Srpaulo * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16214501Srpaulo * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 17214501Srpaulo * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18214501Srpaulo * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19214501Srpaulo * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20214501Srpaulo * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21214501Srpaulo * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22214501Srpaulo * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23214501Srpaulo * SUCH DAMAGE. 24214501Srpaulo */ 25214501Srpaulo 26214501Srpaulo#include <sys/cdefs.h> 27214501Srpaulo__FBSDID("$FreeBSD: stable/11/sys/vm/phys_pager.c 298433 2016-04-21 19:57:40Z pfg $"); 28214501Srpaulo 29214501Srpaulo#include <sys/param.h> 30214501Srpaulo#include <sys/systm.h> 31214501Srpaulo#include <sys/conf.h> 32214501Srpaulo#include <sys/kernel.h> 33214501Srpaulo#include <sys/lock.h> 34214501Srpaulo#include <sys/proc.h> 35214501Srpaulo#include <sys/mutex.h> 36214501Srpaulo#include <sys/mman.h> 37214501Srpaulo#include <sys/rwlock.h> 38214501Srpaulo#include <sys/sysctl.h> 39214501Srpaulo 40214501Srpaulo#include <vm/vm.h> 41214501Srpaulo#include <vm/vm_param.h> 42214501Srpaulo#include <vm/vm_object.h> 43214501Srpaulo#include <vm/vm_page.h> 44214501Srpaulo#include <vm/vm_pager.h> 45214501Srpaulo 46214501Srpaulo/* list of phys pager objects */ 47214501Srpaulostatic struct pagerlst phys_pager_object_list; 48214501Srpaulo/* protect access to phys_pager_object_list */ 49214501Srpaulostatic struct mtx phys_pager_mtx; 50214501Srpaulo 51214501Srpaulostatic void 52214501Srpaulophys_pager_init(void) 53214501Srpaulo{ 54214501Srpaulo 55214501Srpaulo TAILQ_INIT(&phys_pager_object_list); 56214501Srpaulo mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF); 57214501Srpaulo} 58214501Srpaulo 59214501Srpaulo/* 60214501Srpaulo * MPSAFE 61214501Srpaulo */ 62214501Srpaulostatic vm_object_t 63214501Srpaulophys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 64214501Srpaulo vm_ooffset_t foff, struct ucred *cred) 65214501Srpaulo{ 66214501Srpaulo vm_object_t object, object1; 67214501Srpaulo vm_pindex_t pindex; 68214501Srpaulo 69214501Srpaulo /* 70214501Srpaulo * Offset should be page aligned. 71214501Srpaulo */ 72214501Srpaulo if (foff & PAGE_MASK) 73214501Srpaulo return (NULL); 74214501Srpaulo 75214501Srpaulo pindex = OFF_TO_IDX(foff + PAGE_MASK + size); 76214501Srpaulo 77214501Srpaulo if (handle != NULL) { 78214501Srpaulo mtx_lock(&phys_pager_mtx); 79214501Srpaulo /* 80214501Srpaulo * Look up pager, creating as necessary. 81214501Srpaulo */ 82214501Srpaulo object1 = NULL; 83214501Srpaulo object = vm_pager_object_lookup(&phys_pager_object_list, handle); 84214501Srpaulo if (object == NULL) { 85214501Srpaulo /* 86214501Srpaulo * Allocate object and associate it with the pager. 87214501Srpaulo */ 88214501Srpaulo mtx_unlock(&phys_pager_mtx); 89214501Srpaulo object1 = vm_object_allocate(OBJT_PHYS, pindex); 90214501Srpaulo mtx_lock(&phys_pager_mtx); 91214501Srpaulo object = vm_pager_object_lookup(&phys_pager_object_list, 92214501Srpaulo handle); 93214501Srpaulo if (object != NULL) { 94214501Srpaulo /* 95214501Srpaulo * We raced with other thread while 96214501Srpaulo * allocating object. 97214501Srpaulo */ 98214501Srpaulo if (pindex > object->size) 99214501Srpaulo object->size = pindex; 100214501Srpaulo } else { 101214501Srpaulo object = object1; 102214501Srpaulo object1 = NULL; 103214501Srpaulo object->handle = handle; 104214501Srpaulo TAILQ_INSERT_TAIL(&phys_pager_object_list, object, 105214501Srpaulo pager_object_list); 106214501Srpaulo } 107214501Srpaulo } else { 108214501Srpaulo if (pindex > object->size) 109214501Srpaulo object->size = pindex; 110214501Srpaulo } 111214501Srpaulo mtx_unlock(&phys_pager_mtx); 112214501Srpaulo vm_object_deallocate(object1); 113214501Srpaulo } else { 114214501Srpaulo object = vm_object_allocate(OBJT_PHYS, pindex); 115214501Srpaulo } 116214501Srpaulo 117214501Srpaulo return (object); 118214501Srpaulo} 119214501Srpaulo 120214501Srpaulo/* 121214501Srpaulo * MPSAFE 122214501Srpaulo */ 123214501Srpaulostatic void 124214501Srpaulophys_pager_dealloc(vm_object_t object) 125214501Srpaulo{ 126214501Srpaulo 127214501Srpaulo if (object->handle != NULL) { 128214501Srpaulo VM_OBJECT_WUNLOCK(object); 129214501Srpaulo mtx_lock(&phys_pager_mtx); 130214501Srpaulo TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list); 131214501Srpaulo mtx_unlock(&phys_pager_mtx); 132214501Srpaulo VM_OBJECT_WLOCK(object); 133214501Srpaulo } 134214501Srpaulo object->handle = NULL; 135214501Srpaulo object->type = OBJT_DEAD; 136214501Srpaulo} 137214501Srpaulo 138214501Srpaulo/* 139214501Srpaulo * Fill as many pages as vm_fault has allocated for us. 140214501Srpaulo */ 141214501Srpaulostatic int 142214501Srpaulophys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, 143214501Srpaulo int *rahead) 144214501Srpaulo{ 145214501Srpaulo int i; 146214501Srpaulo 147214501Srpaulo VM_OBJECT_ASSERT_WLOCKED(object); 148214501Srpaulo for (i = 0; i < count; i++) { 149214501Srpaulo if (m[i]->valid == 0) { 150214501Srpaulo if ((m[i]->flags & PG_ZERO) == 0) 151214501Srpaulo pmap_zero_page(m[i]); 152214501Srpaulo m[i]->valid = VM_PAGE_BITS_ALL; 153214501Srpaulo } 154214501Srpaulo KASSERT(m[i]->valid == VM_PAGE_BITS_ALL, 155214501Srpaulo ("phys_pager_getpages: partially valid page %p", m[i])); 156214501Srpaulo KASSERT(m[i]->dirty == 0, 157214501Srpaulo ("phys_pager_getpages: dirty page %p", m[i])); 158214501Srpaulo } 159214501Srpaulo if (rbehind) 160214501Srpaulo *rbehind = 0; 161214501Srpaulo if (rahead) 162214501Srpaulo *rahead = 0; 163214501Srpaulo return (VM_PAGER_OK); 164214501Srpaulo} 165214501Srpaulo 166214501Srpaulostatic void 167214501Srpaulophys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, 168214501Srpaulo int *rtvals) 169214501Srpaulo{ 170214501Srpaulo 171214501Srpaulo panic("phys_pager_putpage called"); 172214501Srpaulo} 173214501Srpaulo 174214501Srpaulo/* 175214501Srpaulo * Implement a pretty aggressive clustered getpages strategy. Hint that 176214501Srpaulo * everything in an entire 4MB window should be prefaulted at once. 177214501Srpaulo * 178214501Srpaulo * XXX 4MB (1024 slots per page table page) is convenient for x86, 179214501Srpaulo * but may not be for other arches. 180214501Srpaulo */ 181214501Srpaulo#ifndef PHYSCLUSTER 182214501Srpaulo#define PHYSCLUSTER 1024 183214501Srpaulo#endif 184214501Srpaulostatic boolean_t 185214501Srpaulophys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 186214501Srpaulo int *after) 187214501Srpaulo{ 188214501Srpaulo vm_pindex_t base, end; 189214501Srpaulo 190214501Srpaulo base = rounddown2(pindex, PHYSCLUSTER); 191214501Srpaulo end = base + (PHYSCLUSTER - 1); 192214501Srpaulo if (before != NULL) 193214501Srpaulo *before = pindex - base; 194214501Srpaulo if (after != NULL) 195214501Srpaulo *after = end - pindex; 196214501Srpaulo return (TRUE); 197214501Srpaulo} 198214501Srpaulo 199214501Srpaulostruct pagerops physpagerops = { 200214501Srpaulo .pgo_init = phys_pager_init, 201214501Srpaulo .pgo_alloc = phys_pager_alloc, 202214501Srpaulo .pgo_dealloc = phys_pager_dealloc, 203214501Srpaulo .pgo_getpages = phys_pager_getpages, 204214501Srpaulo .pgo_putpages = phys_pager_putpages, 205214501Srpaulo .pgo_haspage = phys_pager_haspage, 206214501Srpaulo}; 207214501Srpaulo