1139825Simp/*- 260757Speter * Copyright (c) 2000 Peter Wemm 360757Speter * 460757Speter * Redistribution and use in source and binary forms, with or without 560757Speter * modification, are permitted provided that the following conditions 660757Speter * are met: 760757Speter * 1. Redistributions of source code must retain the above copyright 860757Speter * notice, this list of conditions and the following disclaimer. 960757Speter * 2. Redistributions in binary form must reproduce the above copyright 1060757Speter * notice, this list of conditions and the following disclaimer in the 1160757Speter * documentation and/or other materials provided with the distribution. 1260757Speter * 1360757Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 1460757Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1560757Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1660757Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 1760757Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1860757Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1960757Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2060757Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2160757Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2260757Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2360757Speter * SUCH DAMAGE. 2460757Speter */ 2560757Speter 26116226Sobrien#include <sys/cdefs.h> 27116226Sobrien__FBSDID("$FreeBSD$"); 28116226Sobrien 2960757Speter#include <sys/param.h> 3060757Speter#include <sys/systm.h> 3160757Speter#include <sys/conf.h> 3275675Salfred#include <sys/kernel.h> 3376166Smarkm#include <sys/lock.h> 3479224Sdillon#include <sys/proc.h> 3576166Smarkm#include <sys/mutex.h> 3660757Speter#include <sys/mman.h> 3760757Speter#include <sys/sysctl.h> 3860757Speter 3960757Speter#include <vm/vm.h> 4060757Speter#include <vm/vm_object.h> 4160757Speter#include <vm/vm_page.h> 4260757Speter#include <vm/vm_pager.h> 4360757Speter 44173180Sremko/* list of phys pager objects */ 4560757Speterstatic struct pagerlst phys_pager_object_list; 4675675Salfred/* protect access to phys_pager_object_list */ 4775675Salfredstatic struct mtx phys_pager_mtx; 4860757Speter 4960757Speterstatic void 5063973Speterphys_pager_init(void) 5160757Speter{ 5263973Speter 5360757Speter TAILQ_INIT(&phys_pager_object_list); 5493818Sjhb mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF); 5560757Speter} 5660757Speter 5798605Salc/* 5898605Salc * MPSAFE 5998605Salc */ 6060757Speterstatic vm_object_t 6160757Speterphys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 62194766Skib vm_ooffset_t foff, struct ucred *cred) 6360757Speter{ 64171887Skib vm_object_t object, object1; 65124133Salc vm_pindex_t pindex; 6660757Speter 6760757Speter /* 6860757Speter * Offset should be page aligned. 6960757Speter */ 7060757Speter if (foff & PAGE_MASK) 7160757Speter return (NULL); 7260757Speter 73124133Salc pindex = OFF_TO_IDX(foff + PAGE_MASK + size); 7460757Speter 7569687Salfred if (handle != NULL) { 76171887Skib mtx_lock(&phys_pager_mtx); 7760757Speter /* 7869687Salfred * Look up pager, creating as necessary. 7960757Speter */ 80171887Skib object1 = NULL; 8169687Salfred object = vm_pager_object_lookup(&phys_pager_object_list, handle); 8269687Salfred if (object == NULL) { 8369687Salfred /* 8469687Salfred * Allocate object and associate it with the pager. 8569687Salfred */ 86171887Skib mtx_unlock(&phys_pager_mtx); 87171887Skib object1 = vm_object_allocate(OBJT_PHYS, pindex); 8875675Salfred mtx_lock(&phys_pager_mtx); 89171887Skib object = vm_pager_object_lookup(&phys_pager_object_list, 90171887Skib handle); 91171887Skib if (object != NULL) { 92171887Skib /* 93171887Skib * We raced with other thread while 94171887Skib * allocating object. 95171887Skib */ 96171887Skib if (pindex > object->size) 97171887Skib object->size = pindex; 98171887Skib } else { 99171887Skib object = object1; 100171887Skib object1 = NULL; 101171887Skib object->handle = handle; 102171887Skib TAILQ_INSERT_TAIL(&phys_pager_object_list, object, 103171887Skib pager_object_list); 104171887Skib } 10569687Salfred } else { 106124133Salc if (pindex > object->size) 107124133Salc object->size = pindex; 10869687Salfred } 109171887Skib mtx_unlock(&phys_pager_mtx); 110171887Skib vm_object_deallocate(object1); 11169687Salfred } else { 112124133Salc object = vm_object_allocate(OBJT_PHYS, pindex); 11360757Speter } 11460757Speter 11560757Speter return (object); 11660757Speter} 11760757Speter 11898605Salc/* 11998605Salc * MPSAFE 12098605Salc */ 12160757Speterstatic void 12263973Speterphys_pager_dealloc(vm_object_t object) 12360757Speter{ 12460757Speter 12575675Salfred if (object->handle != NULL) { 126171887Skib VM_OBJECT_UNLOCK(object); 12775675Salfred mtx_lock(&phys_pager_mtx); 12869687Salfred TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list); 12975675Salfred mtx_unlock(&phys_pager_mtx); 130171887Skib VM_OBJECT_LOCK(object); 13175675Salfred } 13260757Speter} 13360757Speter 134128620Salc/* 135128620Salc * Fill as many pages as vm_fault has allocated for us. 136128620Salc */ 13760757Speterstatic int 13863973Speterphys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) 13960757Speter{ 140128620Salc int i; 14160757Speter 142128620Salc VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 143128633Salc for (i = 0; i < count; i++) { 144128633Salc if (m[i]->valid == 0) { 145128633Salc if ((m[i]->flags & PG_ZERO) == 0) 146128633Salc pmap_zero_page(m[i]); 147128633Salc m[i]->valid = VM_PAGE_BITS_ALL; 148128633Salc } 149128633Salc KASSERT(m[i]->valid == VM_PAGE_BITS_ALL, 150128633Salc ("phys_pager_getpages: partially valid page %p", m[i])); 151194126Salc KASSERT(m[i]->dirty == 0, 152194126Salc ("phys_pager_getpages: dirty page %p", m[i])); 15360757Speter /* The requested page must remain busy, the others not. */ 154207580Skib if (i == reqpage) 155207580Skib vm_page_flash(m[i]); 156207580Skib else 157207580Skib vm_page_wakeup(m[i]); 15860757Speter } 15960757Speter return (VM_PAGER_OK); 16060757Speter} 16160757Speter 16260757Speterstatic void 16363973Speterphys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, 16463973Speter int *rtvals) 16560757Speter{ 16663973Speter 16760757Speter panic("phys_pager_putpage called"); 16860757Speter} 16960757Speter 17060757Speter/* 17160757Speter * Implement a pretty aggressive clustered getpages strategy. Hint that 17260757Speter * everything in an entire 4MB window should be prefaulted at once. 17360757Speter * 17460757Speter * XXX 4MB (1024 slots per page table page) is convenient for x86, 17560757Speter * but may not be for other arches. 17660757Speter */ 17760757Speter#ifndef PHYSCLUSTER 17860757Speter#define PHYSCLUSTER 1024 17960757Speter#endif 18060757Speterstatic boolean_t 18163973Speterphys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 18263973Speter int *after) 18360757Speter{ 18460757Speter vm_pindex_t base, end; 18560757Speter 18660757Speter base = pindex & (~(PHYSCLUSTER - 1)); 18760757Speter end = base + (PHYSCLUSTER - 1); 18860757Speter if (before != NULL) 18960757Speter *before = pindex - base; 19060757Speter if (after != NULL) 19160757Speter *after = end - pindex; 19260757Speter return (TRUE); 19360757Speter} 19463973Speter 19563973Speterstruct pagerops physpagerops = { 196118466Sphk .pgo_init = phys_pager_init, 197118466Sphk .pgo_alloc = phys_pager_alloc, 198118466Sphk .pgo_dealloc = phys_pager_dealloc, 199118466Sphk .pgo_getpages = phys_pager_getpages, 200118466Sphk .pgo_putpages = phys_pager_putpages, 201118466Sphk .pgo_haspage = phys_pager_haspage, 20263973Speter}; 203