device_pager.c revision 171779
1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: head/sys/vm/device_pager.c 171779 2007-08-07 15:36:25Z kib $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 4775675Salfred#include <sys/sx.h> 481541Srgrimes 491541Srgrimes#include <vm/vm.h> 5012662Sdg#include <vm/vm_object.h> 511541Srgrimes#include <vm/vm_page.h> 529507Sdg#include <vm/vm_pager.h> 5392748Sjeff#include <vm/uma.h> 541541Srgrimes 5592727Salfredstatic void dev_pager_init(void); 5692727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 5792727Salfred vm_ooffset_t); 5892727Salfredstatic void dev_pager_dealloc(vm_object_t); 5992727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 6092727Salfredstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, 6192727Salfred boolean_t, int *); 6292727Salfredstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, 6392727Salfred int *); 641541Srgrimes 6512820Sphk/* list of device pager objects */ 6612820Sphkstatic struct pagerlst dev_pager_object_list; 6775675Salfred/* protect list manipulation */ 6875675Salfredstatic struct mtx dev_pager_mtx; 6912820Sphk 7075675Salfred 7192748Sjeffstatic uma_zone_t fakepg_zone; 7212820Sphk 73112569Sjakestatic vm_page_t dev_pager_getfake(vm_paddr_t); 7492727Salfredstatic void dev_pager_putfake(vm_page_t); 75132884Sdfrstatic void dev_pager_updatefake(vm_page_t, vm_paddr_t); 761541Srgrimes 771541Srgrimesstruct pagerops devicepagerops = { 78118466Sphk .pgo_init = dev_pager_init, 79118466Sphk .pgo_alloc = dev_pager_alloc, 80118466Sphk .pgo_dealloc = dev_pager_dealloc, 81118466Sphk .pgo_getpages = dev_pager_getpages, 82118466Sphk .pgo_putpages = dev_pager_putpages, 83118466Sphk .pgo_haspage = dev_pager_haspage, 841541Srgrimes}; 851541Srgrimes 8612820Sphkstatic void 871541Srgrimesdev_pager_init() 881541Srgrimes{ 899507Sdg TAILQ_INIT(&dev_pager_object_list); 9093818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 9192748Sjeff fakepg_zone = uma_zcreate("DP fakepg", sizeof(struct vm_page), 92120739Sjeff NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 93120739Sjeff UMA_ZONE_NOFREE|UMA_ZONE_VM); 941541Srgrimes} 951541Srgrimes 9698630Salc/* 9798630Salc * MPSAFE 9898630Salc */ 9912820Sphkstatic vm_object_t 10040286Sdgdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff) 1011541Srgrimes{ 102130585Sphk struct cdev *dev; 103171779Skib vm_object_t object, object1; 104124133Salc vm_pindex_t pindex; 10541004Sdfr unsigned int npages; 106112569Sjake vm_paddr_t paddr; 107112569Sjake vm_offset_t off; 108135707Sphk struct cdevsw *csw; 1091541Srgrimes 1101541Srgrimes /* 11198630Salc * Offset should be page aligned. 11298630Salc */ 11398630Salc if (foff & PAGE_MASK) 11498630Salc return (NULL); 11598630Salc 11698630Salc size = round_page(size); 117124133Salc pindex = OFF_TO_IDX(foff + size); 11898630Salc 11998630Salc /* 1201541Srgrimes * Make sure this device can be mapped. 1211541Srgrimes */ 12247111Sbde dev = handle; 123135707Sphk csw = dev_refthread(dev); 124135707Sphk if (csw == NULL) 125135707Sphk return (NULL); 1261541Srgrimes 1271541Srgrimes /* 1285455Sdg * Check that the specified range of the device allows the desired 1295455Sdg * protection. 1308876Srgrimes * 1311541Srgrimes * XXX assumes VM_PROT_* == PROT_* 1321541Srgrimes */ 13340286Sdg npages = OFF_TO_IDX(size); 1341541Srgrimes for (off = foff; npages--; off += PAGE_SIZE) 135135707Sphk if ((*csw->d_mmap)(dev, off, &paddr, (int)prot) != 0) { 136135707Sphk dev_relthread(dev); 1375455Sdg return (NULL); 13898630Salc } 1391541Srgrimes 1401541Srgrimes /* 14158634Scharnier * Lock to prevent object creation race condition. 1429507Sdg */ 143171779Skib mtx_lock(&dev_pager_mtx); 1449507Sdg 1459507Sdg /* 1461541Srgrimes * Look up pager, creating as necessary. 1471541Srgrimes */ 148171779Skib object1 = NULL; 1499507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1509507Sdg if (object == NULL) { 1511541Srgrimes /* 1521541Srgrimes * Allocate object and associate it with the pager. 1531541Srgrimes */ 154171779Skib mtx_unlock(&dev_pager_mtx); 155171779Skib object1 = vm_object_allocate(OBJT_DEVICE, pindex); 15675675Salfred mtx_lock(&dev_pager_mtx); 157171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 158171779Skib if (object != NULL) { 159171779Skib /* 160171779Skib * We raced with other thread while allocating object. 161171779Skib */ 162171779Skib if (pindex > object->size) 163171779Skib object->size = pindex; 164171779Skib } else { 165171779Skib object = object1; 166171779Skib object1 = NULL; 167171779Skib object->handle = handle; 168171779Skib TAILQ_INIT(&object->un_pager.devp.devp_pglist); 169171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 170171779Skib pager_object_list); 171171779Skib } 1721541Srgrimes } else { 173124133Salc if (pindex > object->size) 174124133Salc object->size = pindex; 1751541Srgrimes } 176171779Skib mtx_unlock(&dev_pager_mtx); 177135707Sphk dev_relthread(dev); 178171779Skib vm_object_deallocate(object1); 1799507Sdg return (object); 1801541Srgrimes} 1811541Srgrimes 18212820Sphkstatic void 1839507Sdgdev_pager_dealloc(object) 1849507Sdg vm_object_t object; 1851541Srgrimes{ 1861541Srgrimes vm_page_t m; 1871541Srgrimes 188171779Skib VM_OBJECT_UNLOCK(object); 18975675Salfred mtx_lock(&dev_pager_mtx); 1909507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 19175675Salfred mtx_unlock(&dev_pager_mtx); 192171779Skib VM_OBJECT_LOCK(object); 1931541Srgrimes /* 1941541Srgrimes * Free up our fake pages. 1951541Srgrimes */ 19615809Sdyson while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != 0) { 1979507Sdg TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq); 1981541Srgrimes dev_pager_putfake(m); 1991541Srgrimes } 2001541Srgrimes} 2011541Srgrimes 20212820Sphkstatic int 2039507Sdgdev_pager_getpages(object, m, count, reqpage) 2049507Sdg vm_object_t object; 2059507Sdg vm_page_t *m; 2069507Sdg int count; 2079507Sdg int reqpage; 2081541Srgrimes{ 20998824Siedowse vm_pindex_t offset; 210112569Sjake vm_paddr_t paddr; 2111541Srgrimes vm_page_t page; 212130585Sphk struct cdev *dev; 213111462Smux int i, ret; 21412591Sbde int prot; 215135707Sphk struct cdevsw *csw; 2161541Srgrimes 217116793Salc VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 21847111Sbde dev = object->handle; 21942957Sdillon offset = m[reqpage]->pindex; 220116279Salc VM_OBJECT_UNLOCK(object); 221135707Sphk csw = dev_refthread(dev); 222135707Sphk if (csw == NULL) 223135707Sphk panic("dev_pager_getpage: no cdevsw"); 2241541Srgrimes prot = PROT_READ; /* XXX should pass in? */ 2251549Srgrimes 226135707Sphk ret = (*csw->d_mmap)(dev, (vm_offset_t)offset << PAGE_SHIFT, &paddr, prot); 227111462Smux KASSERT(ret == 0, ("dev_pager_getpage: map function returns error")); 228135707Sphk dev_relthread(dev); 229128570Salc 230132884Sdfr if ((m[reqpage]->flags & PG_FICTITIOUS) != 0) { 231132884Sdfr /* 232132884Sdfr * If the passed in reqpage page is a fake page, update it with 233132884Sdfr * the new physical address. 234132884Sdfr */ 235133113Sdfr VM_OBJECT_LOCK(object); 236132884Sdfr dev_pager_updatefake(m[reqpage], paddr); 237132884Sdfr if (count > 1) { 238132884Sdfr vm_page_lock_queues(); 239132884Sdfr for (i = 0; i < count; i++) { 240132884Sdfr if (i != reqpage) 241132884Sdfr vm_page_free(m[i]); 242132884Sdfr } 243132884Sdfr vm_page_unlock_queues(); 244132884Sdfr } 245132884Sdfr } else { 246132884Sdfr /* 247132884Sdfr * Replace the passed in reqpage page with our own fake page and 248132884Sdfr * free up the all of the original pages. 249132884Sdfr */ 250132884Sdfr page = dev_pager_getfake(paddr); 251132884Sdfr VM_OBJECT_LOCK(object); 252132884Sdfr TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq); 253132884Sdfr vm_page_lock_queues(); 254132884Sdfr for (i = 0; i < count; i++) 255132884Sdfr vm_page_free(m[i]); 256132884Sdfr vm_page_unlock_queues(); 257132884Sdfr vm_page_insert(page, object, offset); 258132884Sdfr m[reqpage] = page; 259132884Sdfr } 2601541Srgrimes 2615455Sdg return (VM_PAGER_OK); 2621541Srgrimes} 2631541Srgrimes 26443129Sdillonstatic void 2659507Sdgdev_pager_putpages(object, m, count, sync, rtvals) 2669507Sdg vm_object_t object; 2679507Sdg vm_page_t *m; 2689507Sdg int count; 2691541Srgrimes boolean_t sync; 2709507Sdg int *rtvals; 2711541Srgrimes{ 2721541Srgrimes panic("dev_pager_putpage called"); 2731541Srgrimes} 2741541Srgrimes 27512820Sphkstatic boolean_t 27612767Sdysondev_pager_haspage(object, pindex, before, after) 2779507Sdg vm_object_t object; 27812767Sdyson vm_pindex_t pindex; 2799507Sdg int *before; 2809507Sdg int *after; 2811541Srgrimes{ 2829507Sdg if (before != NULL) 2839507Sdg *before = 0; 2849507Sdg if (after != NULL) 2859507Sdg *after = 0; 2865455Sdg return (TRUE); 2871541Srgrimes} 2881541Srgrimes 289147262Salc/* 290147262Salc * Instantiate a fictitious page. Unlike physical memory pages, only 291147262Salc * the machine-independent fields must be initialized. 292147262Salc */ 2931541Srgrimesstatic vm_page_t 2941541Srgrimesdev_pager_getfake(paddr) 295112569Sjake vm_paddr_t paddr; 2961541Srgrimes{ 2971541Srgrimes vm_page_t m; 2981541Srgrimes 299111119Simp m = uma_zalloc(fakepg_zone, M_WAITOK); 3001549Srgrimes 301163604Salc m->flags = PG_FICTITIOUS; 302163604Salc m->oflags = VPO_BUSY; 3039507Sdg m->valid = VM_PAGE_BITS_ALL; 3045455Sdg m->dirty = 0; 3055455Sdg m->busy = 0; 30613490Sdyson m->queue = PQ_NONE; 30740557Sdg m->object = NULL; 3081549Srgrimes 3091549Srgrimes m->wire_count = 1; 31014430Sdyson m->hold_count = 0; 3111541Srgrimes m->phys_addr = paddr; 3121549Srgrimes 3135455Sdg return (m); 3141541Srgrimes} 3151541Srgrimes 3161541Srgrimesstatic void 3171541Srgrimesdev_pager_putfake(m) 3181541Srgrimes vm_page_t m; 3191541Srgrimes{ 3201541Srgrimes if (!(m->flags & PG_FICTITIOUS)) 3211541Srgrimes panic("dev_pager_putfake: bad page"); 32292748Sjeff uma_zfree(fakepg_zone, m); 3231541Srgrimes} 324132884Sdfr 325132884Sdfrstatic void 326132884Sdfrdev_pager_updatefake(m, paddr) 327132884Sdfr vm_page_t m; 328132884Sdfr vm_paddr_t paddr; 329132884Sdfr{ 330132884Sdfr if (!(m->flags & PG_FICTITIOUS)) 331132884Sdfr panic("dev_pager_updatefake: bad page"); 332132884Sdfr m->phys_addr = paddr; 333133113Sdfr m->valid = VM_PAGE_BITS_ALL; 334132884Sdfr} 335