device_pager.c revision 229383
1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: stable/9/sys/vm/device_pager.c 229383 2012-01-03 10:36:38Z kib $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 4775675Salfred#include <sys/sx.h> 481541Srgrimes 491541Srgrimes#include <vm/vm.h> 5012662Sdg#include <vm/vm_object.h> 511541Srgrimes#include <vm/vm_page.h> 529507Sdg#include <vm/vm_pager.h> 5392748Sjeff#include <vm/uma.h> 541541Srgrimes 5592727Salfredstatic void dev_pager_init(void); 5692727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 57194766Skib vm_ooffset_t, struct ucred *); 5892727Salfredstatic void dev_pager_dealloc(vm_object_t); 5992727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 6092727Salfredstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, 6192727Salfred boolean_t, int *); 6292727Salfredstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, 6392727Salfred int *); 641541Srgrimes 6512820Sphk/* list of device pager objects */ 6612820Sphkstatic struct pagerlst dev_pager_object_list; 6775675Salfred/* protect list manipulation */ 6875675Salfredstatic struct mtx dev_pager_mtx; 6912820Sphk 701541Srgrimesstruct pagerops devicepagerops = { 71118466Sphk .pgo_init = dev_pager_init, 72118466Sphk .pgo_alloc = dev_pager_alloc, 73118466Sphk .pgo_dealloc = dev_pager_dealloc, 74118466Sphk .pgo_getpages = dev_pager_getpages, 75118466Sphk .pgo_putpages = dev_pager_putpages, 76118466Sphk .pgo_haspage = dev_pager_haspage, 771541Srgrimes}; 781541Srgrimes 79229383Skibstatic int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 80229383Skib vm_ooffset_t foff, struct ucred *cred, u_short *color); 81229383Skibstatic void old_dev_pager_dtor(void *handle); 82229383Skibstatic int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 83229383Skib int prot, vm_page_t *mres); 84229383Skib 85229383Skibstatic struct cdev_pager_ops old_dev_pager_ops = { 86229383Skib .cdev_pg_ctor = old_dev_pager_ctor, 87229383Skib .cdev_pg_dtor = old_dev_pager_dtor, 88229383Skib .cdev_pg_fault = old_dev_pager_fault 89229383Skib}; 90229383Skib 9112820Sphkstatic void 921541Srgrimesdev_pager_init() 931541Srgrimes{ 949507Sdg TAILQ_INIT(&dev_pager_object_list); 9593818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 961541Srgrimes} 971541Srgrimes 98229383Skibvm_object_t 99229383Skibcdev_pager_lookup(void *handle) 1001541Srgrimes{ 101229383Skib vm_object_t object; 102229383Skib 103229383Skib mtx_lock(&dev_pager_mtx); 104229383Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 105229383Skib vm_object_reference(object); 106229383Skib mtx_unlock(&dev_pager_mtx); 107229383Skib return (object); 108229383Skib} 109229383Skib 110229383Skibvm_object_t 111229383Skibcdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops, 112229383Skib vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 113229383Skib{ 114171779Skib vm_object_t object, object1; 115124133Salc vm_pindex_t pindex; 116229383Skib u_short color; 1171541Srgrimes 118229383Skib if (tp != OBJT_DEVICE) 119229383Skib return (NULL); 120229383Skib 1211541Srgrimes /* 12298630Salc * Offset should be page aligned. 12398630Salc */ 12498630Salc if (foff & PAGE_MASK) 12598630Salc return (NULL); 12698630Salc 12798630Salc size = round_page(size); 128124133Salc pindex = OFF_TO_IDX(foff + size); 12998630Salc 130229383Skib if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0) 131135707Sphk return (NULL); 132171779Skib mtx_lock(&dev_pager_mtx); 1339507Sdg 1349507Sdg /* 1351541Srgrimes * Look up pager, creating as necessary. 1361541Srgrimes */ 137171779Skib object1 = NULL; 1389507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1399507Sdg if (object == NULL) { 1401541Srgrimes /* 141179074Salc * Allocate object and associate it with the pager. Initialize 142179074Salc * the object's pg_color based upon the physical address of the 143179074Salc * device's memory. 1441541Srgrimes */ 145171779Skib mtx_unlock(&dev_pager_mtx); 146229383Skib object1 = vm_object_allocate(tp, pindex); 147179074Salc object1->flags |= OBJ_COLORED; 148229383Skib object1->pg_color = color; 149229383Skib object1->handle = handle; 150229383Skib object1->un_pager.devp.ops = ops; 151224522Skib TAILQ_INIT(&object1->un_pager.devp.devp_pglist); 15275675Salfred mtx_lock(&dev_pager_mtx); 153171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 154171779Skib if (object != NULL) { 155171779Skib /* 156171779Skib * We raced with other thread while allocating object. 157171779Skib */ 158171779Skib if (pindex > object->size) 159171779Skib object->size = pindex; 160171779Skib } else { 161171779Skib object = object1; 162171779Skib object1 = NULL; 163171779Skib object->handle = handle; 164171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 165171779Skib pager_object_list); 166229383Skib KASSERT(object->type == tp, 167229383Skib ("Inconsistent device pager type %p %d", object, tp)); 168171779Skib } 1691541Srgrimes } else { 170124133Salc if (pindex > object->size) 171124133Salc object->size = pindex; 1721541Srgrimes } 173171779Skib mtx_unlock(&dev_pager_mtx); 174224522Skib if (object1 != NULL) { 175224522Skib object1->handle = object1; 176224522Skib mtx_lock(&dev_pager_mtx); 177224522Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object1, 178224522Skib pager_object_list); 179224522Skib mtx_unlock(&dev_pager_mtx); 180224522Skib vm_object_deallocate(object1); 181224522Skib } 1829507Sdg return (object); 1831541Srgrimes} 1841541Srgrimes 185229383Skibstatic vm_object_t 186229383Skibdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 187229383Skib vm_ooffset_t foff, struct ucred *cred) 188229383Skib{ 189229383Skib 190229383Skib return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops, 191229383Skib size, prot, foff, cred)); 192229383Skib} 193229383Skib 194229383Skibvoid 195229383Skibcdev_pager_free_page(vm_object_t object, vm_page_t m) 196229383Skib{ 197229383Skib 198229383Skib VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 199229383Skib TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq); 200229383Skib vm_page_putfake(m); 201229383Skib} 202229383Skib 20312820Sphkstatic void 2049507Sdgdev_pager_dealloc(object) 2059507Sdg vm_object_t object; 2061541Srgrimes{ 2071541Srgrimes vm_page_t m; 2081541Srgrimes 209171779Skib VM_OBJECT_UNLOCK(object); 210229383Skib object->un_pager.devp.ops->cdev_pg_dtor(object->handle); 211229383Skib 21275675Salfred mtx_lock(&dev_pager_mtx); 2139507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 21475675Salfred mtx_unlock(&dev_pager_mtx); 215171779Skib VM_OBJECT_LOCK(object); 2161541Srgrimes /* 2171541Srgrimes * Free up our fake pages. 2181541Srgrimes */ 219229383Skib while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != NULL) 220229383Skib cdev_pager_free_page(object, m); 221229383Skib} 222229383Skib 223229383Skibstatic int 224229383Skibdev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage) 225229383Skib{ 226229383Skib int error, i; 227229383Skib 228229383Skib VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 229229383Skib error = object->un_pager.devp.ops->cdev_pg_fault(object, 230229383Skib IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]); 231229383Skib 232229383Skib VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 233229383Skib 234229383Skib for (i = 0; i < count; i++) { 235229383Skib if (i != reqpage) { 236229383Skib vm_page_lock(ma[i]); 237229383Skib vm_page_free(ma[i]); 238229383Skib vm_page_unlock(ma[i]); 239229383Skib } 2401541Srgrimes } 241229383Skib 242229383Skib if (error == VM_PAGER_OK) { 243229383Skib TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, 244229383Skib ma[reqpage], pageq); 245229383Skib } 246229383Skib 247229383Skib return (error); 2481541Srgrimes} 2491541Srgrimes 25012820Sphkstatic int 251229383Skibold_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, 252229383Skib vm_page_t *mres) 2531541Srgrimes{ 254229383Skib vm_pindex_t pidx; 255112569Sjake vm_paddr_t paddr; 256195649Salc vm_page_t m_paddr, page; 257130585Sphk struct cdev *dev; 258135707Sphk struct cdevsw *csw; 259229383Skib struct file *fpop; 260183383Skib struct thread *td; 261229383Skib vm_memattr_t memattr; 262229383Skib int ref, ret; 2631541Srgrimes 264229383Skib pidx = OFF_TO_IDX(offset); 265195649Salc memattr = object->memattr; 266229383Skib 267116279Salc VM_OBJECT_UNLOCK(object); 268229383Skib 269229383Skib dev = object->handle; 270210923Skib csw = dev_refthread(dev, &ref); 271223823Sattilio if (csw == NULL) { 272223823Sattilio VM_OBJECT_LOCK(object); 273223823Sattilio return (VM_PAGER_FAIL); 274223823Sattilio } 275183383Skib td = curthread; 276183383Skib fpop = td->td_fpop; 277183383Skib td->td_fpop = NULL; 278229383Skib ret = csw->d_mmap(dev, offset, &paddr, prot, &memattr); 279183383Skib td->td_fpop = fpop; 280210923Skib dev_relthread(dev, ref); 281229383Skib if (ret != 0) { 282229383Skib printf( 283229383Skib "WARNING: dev_pager_getpage: map function returns error %d", ret); 284229383Skib VM_OBJECT_LOCK(object); 285229383Skib return (VM_PAGER_FAIL); 286229383Skib } 287229383Skib 288195649Salc /* If "paddr" is a real page, perform a sanity check on "memattr". */ 289195649Salc if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL && 290195649Salc pmap_page_get_memattr(m_paddr) != memattr) { 291195649Salc memattr = pmap_page_get_memattr(m_paddr); 292195649Salc printf( 293195649Salc "WARNING: A device driver has set \"memattr\" inconsistently.\n"); 294195649Salc } 295229383Skib if (((*mres)->flags & PG_FICTITIOUS) != 0) { 296132884Sdfr /* 297229383Skib * If the passed in result page is a fake page, update it with 298132884Sdfr * the new physical address. 299132884Sdfr */ 300229383Skib page = *mres; 301133113Sdfr VM_OBJECT_LOCK(object); 302219476Salc vm_page_updatefake(page, paddr, memattr); 303132884Sdfr } else { 304132884Sdfr /* 305132884Sdfr * Replace the passed in reqpage page with our own fake page and 306132884Sdfr * free up the all of the original pages. 307132884Sdfr */ 308219476Salc page = vm_page_getfake(paddr, memattr); 309132884Sdfr VM_OBJECT_LOCK(object); 310229383Skib vm_page_lock(*mres); 311229383Skib vm_page_free(*mres); 312229383Skib vm_page_unlock(*mres); 313229383Skib *mres = page; 314229383Skib vm_page_insert(page, object, pidx); 315132884Sdfr } 316194642Salc page->valid = VM_PAGE_BITS_ALL; 3175455Sdg return (VM_PAGER_OK); 3181541Srgrimes} 3191541Srgrimes 32043129Sdillonstatic void 3219507Sdgdev_pager_putpages(object, m, count, sync, rtvals) 3229507Sdg vm_object_t object; 3239507Sdg vm_page_t *m; 3249507Sdg int count; 3251541Srgrimes boolean_t sync; 3269507Sdg int *rtvals; 3271541Srgrimes{ 328229383Skib 3291541Srgrimes panic("dev_pager_putpage called"); 3301541Srgrimes} 3311541Srgrimes 33212820Sphkstatic boolean_t 33312767Sdysondev_pager_haspage(object, pindex, before, after) 3349507Sdg vm_object_t object; 33512767Sdyson vm_pindex_t pindex; 3369507Sdg int *before; 3379507Sdg int *after; 3381541Srgrimes{ 3399507Sdg if (before != NULL) 3409507Sdg *before = 0; 3419507Sdg if (after != NULL) 3429507Sdg *after = 0; 3435455Sdg return (TRUE); 3441541Srgrimes} 345229383Skib 346229383Skibstatic int 347229383Skibold_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 348229383Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 349229383Skib{ 350229383Skib struct cdev *dev; 351229383Skib struct cdevsw *csw; 352229383Skib vm_memattr_t dummy; 353229383Skib vm_ooffset_t off; 354229383Skib vm_paddr_t paddr; 355229383Skib unsigned int npages; 356229383Skib int ref; 357229383Skib 358229383Skib /* 359229383Skib * Make sure this device can be mapped. 360229383Skib */ 361229383Skib dev = handle; 362229383Skib csw = dev_refthread(dev, &ref); 363229383Skib if (csw == NULL) 364229383Skib return (ENXIO); 365229383Skib 366229383Skib /* 367229383Skib * Check that the specified range of the device allows the desired 368229383Skib * protection. 369229383Skib * 370229383Skib * XXX assumes VM_PROT_* == PROT_* 371229383Skib */ 372229383Skib npages = OFF_TO_IDX(size); 373229383Skib for (off = foff; npages--; off += PAGE_SIZE) { 374229383Skib if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) { 375229383Skib dev_relthread(dev, ref); 376229383Skib return (EINVAL); 377229383Skib } 378229383Skib } 379229383Skib 380229383Skib dev_ref(dev); 381229383Skib dev_relthread(dev, ref); 382229383Skib *color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE); 383229383Skib return (0); 384229383Skib} 385229383Skib 386229383Skibstatic void 387229383Skibold_dev_pager_dtor(void *handle) 388229383Skib{ 389229383Skib 390229383Skib dev_rel(handle); 391229383Skib} 392