device_pager.c revision 254141
1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: head/sys/vm/device_pager.c 254141 2013-08-09 11:28:55Z attilio $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 47248084Sattilio#include <sys/rwlock.h> 4875675Salfred#include <sys/sx.h> 491541Srgrimes 501541Srgrimes#include <vm/vm.h> 51239065Skib#include <vm/vm_param.h> 5212662Sdg#include <vm/vm_object.h> 531541Srgrimes#include <vm/vm_page.h> 549507Sdg#include <vm/vm_pager.h> 55243132Skib#include <vm/vm_phys.h> 5692748Sjeff#include <vm/uma.h> 571541Srgrimes 5892727Salfredstatic void dev_pager_init(void); 5992727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 60194766Skib vm_ooffset_t, struct ucred *); 6192727Salfredstatic void dev_pager_dealloc(vm_object_t); 6292727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 6392727Salfredstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, 6492727Salfred boolean_t, int *); 6592727Salfredstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, 6692727Salfred int *); 67235375Skibstatic void dev_pager_free_page(vm_object_t object, vm_page_t m); 681541Srgrimes 6912820Sphk/* list of device pager objects */ 7012820Sphkstatic struct pagerlst dev_pager_object_list; 7175675Salfred/* protect list manipulation */ 7275675Salfredstatic struct mtx dev_pager_mtx; 7312820Sphk 741541Srgrimesstruct pagerops devicepagerops = { 75118466Sphk .pgo_init = dev_pager_init, 76118466Sphk .pgo_alloc = dev_pager_alloc, 77118466Sphk .pgo_dealloc = dev_pager_dealloc, 78118466Sphk .pgo_getpages = dev_pager_getpages, 79118466Sphk .pgo_putpages = dev_pager_putpages, 80118466Sphk .pgo_haspage = dev_pager_haspage, 811541Srgrimes}; 821541Srgrimes 83235375Skibstruct pagerops mgtdevicepagerops = { 84235375Skib .pgo_alloc = dev_pager_alloc, 85235375Skib .pgo_dealloc = dev_pager_dealloc, 86235375Skib .pgo_getpages = dev_pager_getpages, 87235375Skib .pgo_putpages = dev_pager_putpages, 88235375Skib .pgo_haspage = dev_pager_haspage, 89235375Skib}; 90235375Skib 91227530Skibstatic int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 92227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color); 93227530Skibstatic void old_dev_pager_dtor(void *handle); 94227530Skibstatic int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 95227530Skib int prot, vm_page_t *mres); 96227530Skib 97227530Skibstatic struct cdev_pager_ops old_dev_pager_ops = { 98227530Skib .cdev_pg_ctor = old_dev_pager_ctor, 99227530Skib .cdev_pg_dtor = old_dev_pager_dtor, 100227530Skib .cdev_pg_fault = old_dev_pager_fault 101227530Skib}; 102227530Skib 10312820Sphkstatic void 1041541Srgrimesdev_pager_init() 1051541Srgrimes{ 1069507Sdg TAILQ_INIT(&dev_pager_object_list); 10793818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 1081541Srgrimes} 1091541Srgrimes 110227530Skibvm_object_t 111227530Skibcdev_pager_lookup(void *handle) 1121541Srgrimes{ 113227530Skib vm_object_t object; 114227530Skib 115227530Skib mtx_lock(&dev_pager_mtx); 116227530Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 117227530Skib mtx_unlock(&dev_pager_mtx); 118227530Skib return (object); 119227530Skib} 120227530Skib 121227530Skibvm_object_t 122227530Skibcdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops, 123227530Skib vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 124227530Skib{ 125171779Skib vm_object_t object, object1; 126124133Salc vm_pindex_t pindex; 127227530Skib u_short color; 1281541Srgrimes 129235375Skib if (tp != OBJT_DEVICE && tp != OBJT_MGTDEVICE) 130227530Skib return (NULL); 131227530Skib 1321541Srgrimes /* 13398630Salc * Offset should be page aligned. 13498630Salc */ 13598630Salc if (foff & PAGE_MASK) 13698630Salc return (NULL); 13798630Salc 13898630Salc size = round_page(size); 139124133Salc pindex = OFF_TO_IDX(foff + size); 14098630Salc 141227530Skib if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0) 142135707Sphk return (NULL); 143171779Skib mtx_lock(&dev_pager_mtx); 1449507Sdg 1459507Sdg /* 1461541Srgrimes * Look up pager, creating as necessary. 1471541Srgrimes */ 148171779Skib object1 = NULL; 1499507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1509507Sdg if (object == NULL) { 1511541Srgrimes /* 152179074Salc * Allocate object and associate it with the pager. Initialize 153179074Salc * the object's pg_color based upon the physical address of the 154179074Salc * device's memory. 1551541Srgrimes */ 156171779Skib mtx_unlock(&dev_pager_mtx); 157227530Skib object1 = vm_object_allocate(tp, pindex); 158179074Salc object1->flags |= OBJ_COLORED; 159227530Skib object1->pg_color = color; 160227530Skib object1->handle = handle; 161227530Skib object1->un_pager.devp.ops = ops; 162245226Sken object1->un_pager.devp.dev = handle; 163224522Skib TAILQ_INIT(&object1->un_pager.devp.devp_pglist); 16475675Salfred mtx_lock(&dev_pager_mtx); 165171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 166171779Skib if (object != NULL) { 167171779Skib /* 168171779Skib * We raced with other thread while allocating object. 169171779Skib */ 170171779Skib if (pindex > object->size) 171171779Skib object->size = pindex; 172171779Skib } else { 173171779Skib object = object1; 174171779Skib object1 = NULL; 175171779Skib object->handle = handle; 176171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 177171779Skib pager_object_list); 178227530Skib KASSERT(object->type == tp, 179227530Skib ("Inconsistent device pager type %p %d", object, tp)); 180171779Skib } 1811541Srgrimes } else { 182124133Salc if (pindex > object->size) 183124133Salc object->size = pindex; 1841541Srgrimes } 185171779Skib mtx_unlock(&dev_pager_mtx); 186224522Skib if (object1 != NULL) { 187224522Skib object1->handle = object1; 188224522Skib mtx_lock(&dev_pager_mtx); 189224522Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object1, 190224522Skib pager_object_list); 191224522Skib mtx_unlock(&dev_pager_mtx); 192224522Skib vm_object_deallocate(object1); 193224522Skib } 1949507Sdg return (object); 1951541Srgrimes} 1961541Srgrimes 197227530Skibstatic vm_object_t 198227530Skibdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 199227530Skib vm_ooffset_t foff, struct ucred *cred) 200227530Skib{ 201227530Skib 202227530Skib return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops, 203227530Skib size, prot, foff, cred)); 204227530Skib} 205227530Skib 206227530Skibvoid 207227530Skibcdev_pager_free_page(vm_object_t object, vm_page_t m) 208227530Skib{ 209227530Skib 210248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 211235375Skib if (object->type == OBJT_MGTDEVICE) { 212235375Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("unmanaged %p", m)); 213235375Skib pmap_remove_all(m); 214235375Skib vm_page_lock(m); 215235375Skib vm_page_remove(m); 216235375Skib vm_page_unlock(m); 217235375Skib } else if (object->type == OBJT_DEVICE) 218235375Skib dev_pager_free_page(object, m); 219235375Skib} 220235375Skib 221235375Skibstatic void 222235375Skibdev_pager_free_page(vm_object_t object, vm_page_t m) 223235375Skib{ 224235375Skib 225248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 226235375Skib KASSERT((object->type == OBJT_DEVICE && 227235375Skib (m->oflags & VPO_UNMANAGED) != 0), 228235375Skib ("Managed device or page obj %p m %p", object, m)); 229227530Skib TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq); 230227530Skib vm_page_putfake(m); 231227530Skib} 232227530Skib 23312820Sphkstatic void 2349507Sdgdev_pager_dealloc(object) 2359507Sdg vm_object_t object; 2361541Srgrimes{ 2371541Srgrimes vm_page_t m; 2381541Srgrimes 239248084Sattilio VM_OBJECT_WUNLOCK(object); 240245226Sken object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.dev); 241227530Skib 24275675Salfred mtx_lock(&dev_pager_mtx); 2439507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 24475675Salfred mtx_unlock(&dev_pager_mtx); 245248084Sattilio VM_OBJECT_WLOCK(object); 246235375Skib 247235375Skib if (object->type == OBJT_DEVICE) { 248235375Skib /* 249235375Skib * Free up our fake pages. 250235375Skib */ 251235375Skib while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) 252235375Skib != NULL) 253235375Skib dev_pager_free_page(object, m); 254235375Skib } 255227530Skib} 256227530Skib 257227530Skibstatic int 258227530Skibdev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage) 259227530Skib{ 260227530Skib int error, i; 261227530Skib 262248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 263227530Skib error = object->un_pager.devp.ops->cdev_pg_fault(object, 264227530Skib IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]); 265227530Skib 266248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 267227530Skib 268227530Skib for (i = 0; i < count; i++) { 269227530Skib if (i != reqpage) { 270227530Skib vm_page_lock(ma[i]); 271227530Skib vm_page_free(ma[i]); 272227530Skib vm_page_unlock(ma[i]); 273227530Skib } 2741541Srgrimes } 275227530Skib 276227530Skib if (error == VM_PAGER_OK) { 277235375Skib KASSERT((object->type == OBJT_DEVICE && 278235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) != 0) || 279235375Skib (object->type == OBJT_MGTDEVICE && 280235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) == 0), 281235375Skib ("Wrong page type %p %p", ma[reqpage], object)); 282235375Skib if (object->type == OBJT_DEVICE) { 283235375Skib TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, 284235375Skib ma[reqpage], pageq); 285235375Skib } 286227530Skib } 287227530Skib 288227530Skib return (error); 2891541Srgrimes} 2901541Srgrimes 29112820Sphkstatic int 292227530Skibold_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, 293227530Skib vm_page_t *mres) 2941541Srgrimes{ 295227530Skib vm_pindex_t pidx; 296112569Sjake vm_paddr_t paddr; 297195649Salc vm_page_t m_paddr, page; 298130585Sphk struct cdev *dev; 299135707Sphk struct cdevsw *csw; 300227530Skib struct file *fpop; 301183383Skib struct thread *td; 302227530Skib vm_memattr_t memattr; 303227530Skib int ref, ret; 3041541Srgrimes 305227530Skib pidx = OFF_TO_IDX(offset); 306195649Salc memattr = object->memattr; 307227530Skib 308248084Sattilio VM_OBJECT_WUNLOCK(object); 309227530Skib 310227530Skib dev = object->handle; 311210923Skib csw = dev_refthread(dev, &ref); 312223823Sattilio if (csw == NULL) { 313248084Sattilio VM_OBJECT_WLOCK(object); 314223823Sattilio return (VM_PAGER_FAIL); 315223823Sattilio } 316183383Skib td = curthread; 317183383Skib fpop = td->td_fpop; 318183383Skib td->td_fpop = NULL; 319227530Skib ret = csw->d_mmap(dev, offset, &paddr, prot, &memattr); 320183383Skib td->td_fpop = fpop; 321210923Skib dev_relthread(dev, ref); 322227530Skib if (ret != 0) { 323227530Skib printf( 324227530Skib "WARNING: dev_pager_getpage: map function returns error %d", ret); 325248084Sattilio VM_OBJECT_WLOCK(object); 326227530Skib return (VM_PAGER_FAIL); 327227530Skib } 328227530Skib 329195649Salc /* If "paddr" is a real page, perform a sanity check on "memattr". */ 330195649Salc if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL && 331195649Salc pmap_page_get_memattr(m_paddr) != memattr) { 332195649Salc memattr = pmap_page_get_memattr(m_paddr); 333195649Salc printf( 334195649Salc "WARNING: A device driver has set \"memattr\" inconsistently.\n"); 335195649Salc } 336227530Skib if (((*mres)->flags & PG_FICTITIOUS) != 0) { 337132884Sdfr /* 338227530Skib * If the passed in result page is a fake page, update it with 339132884Sdfr * the new physical address. 340132884Sdfr */ 341227530Skib page = *mres; 342248084Sattilio VM_OBJECT_WLOCK(object); 343219476Salc vm_page_updatefake(page, paddr, memattr); 344132884Sdfr } else { 345132884Sdfr /* 346132884Sdfr * Replace the passed in reqpage page with our own fake page and 347132884Sdfr * free up the all of the original pages. 348132884Sdfr */ 349219476Salc page = vm_page_getfake(paddr, memattr); 350248084Sattilio VM_OBJECT_WLOCK(object); 351254141Sattilio if (vm_page_replace(page, object, (*mres)->pindex) != *mres) 352254141Sattilio panic("old_dev_pager_fault: invalid page replacement"); 353227530Skib vm_page_lock(*mres); 354227530Skib vm_page_free(*mres); 355227530Skib vm_page_unlock(*mres); 356227530Skib *mres = page; 357132884Sdfr } 358194642Salc page->valid = VM_PAGE_BITS_ALL; 3595455Sdg return (VM_PAGER_OK); 3601541Srgrimes} 3611541Srgrimes 36243129Sdillonstatic void 3639507Sdgdev_pager_putpages(object, m, count, sync, rtvals) 3649507Sdg vm_object_t object; 3659507Sdg vm_page_t *m; 3669507Sdg int count; 3671541Srgrimes boolean_t sync; 3689507Sdg int *rtvals; 3691541Srgrimes{ 370227530Skib 3711541Srgrimes panic("dev_pager_putpage called"); 3721541Srgrimes} 3731541Srgrimes 37412820Sphkstatic boolean_t 37512767Sdysondev_pager_haspage(object, pindex, before, after) 3769507Sdg vm_object_t object; 37712767Sdyson vm_pindex_t pindex; 3789507Sdg int *before; 3799507Sdg int *after; 3801541Srgrimes{ 3819507Sdg if (before != NULL) 3829507Sdg *before = 0; 3839507Sdg if (after != NULL) 3849507Sdg *after = 0; 3855455Sdg return (TRUE); 3861541Srgrimes} 387227530Skib 388227530Skibstatic int 389227530Skibold_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 390227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 391227530Skib{ 392227530Skib struct cdev *dev; 393227530Skib struct cdevsw *csw; 394227530Skib vm_memattr_t dummy; 395227530Skib vm_ooffset_t off; 396227530Skib vm_paddr_t paddr; 397227530Skib unsigned int npages; 398227530Skib int ref; 399227530Skib 400227530Skib /* 401227530Skib * Make sure this device can be mapped. 402227530Skib */ 403227530Skib dev = handle; 404227530Skib csw = dev_refthread(dev, &ref); 405227530Skib if (csw == NULL) 406227530Skib return (ENXIO); 407227530Skib 408227530Skib /* 409227530Skib * Check that the specified range of the device allows the desired 410227530Skib * protection. 411227530Skib * 412227530Skib * XXX assumes VM_PROT_* == PROT_* 413227530Skib */ 414227530Skib npages = OFF_TO_IDX(size); 415227530Skib for (off = foff; npages--; off += PAGE_SIZE) { 416227530Skib if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) { 417227530Skib dev_relthread(dev, ref); 418227530Skib return (EINVAL); 419227530Skib } 420227530Skib } 421227530Skib 422227530Skib dev_ref(dev); 423227530Skib dev_relthread(dev, ref); 424227530Skib *color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE); 425227530Skib return (0); 426227530Skib} 427227530Skib 428227530Skibstatic void 429227530Skibold_dev_pager_dtor(void *handle) 430227530Skib{ 431227530Skib 432227530Skib dev_rel(handle); 433227530Skib} 434