device_pager.c revision 284100
1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: stable/10/sys/vm/device_pager.c 284100 2015-06-06 20:37:40Z jhb $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 47248084Sattilio#include <sys/rwlock.h> 4875675Salfred#include <sys/sx.h> 491541Srgrimes 501541Srgrimes#include <vm/vm.h> 51239065Skib#include <vm/vm_param.h> 5212662Sdg#include <vm/vm_object.h> 531541Srgrimes#include <vm/vm_page.h> 549507Sdg#include <vm/vm_pager.h> 55243132Skib#include <vm/vm_phys.h> 5692748Sjeff#include <vm/uma.h> 571541Srgrimes 5892727Salfredstatic void dev_pager_init(void); 5992727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 60194766Skib vm_ooffset_t, struct ucred *); 6192727Salfredstatic void dev_pager_dealloc(vm_object_t); 6292727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 6392727Salfredstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, 6492727Salfred boolean_t, int *); 6592727Salfredstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, 6692727Salfred int *); 67235375Skibstatic void dev_pager_free_page(vm_object_t object, vm_page_t m); 681541Srgrimes 6912820Sphk/* list of device pager objects */ 7012820Sphkstatic struct pagerlst dev_pager_object_list; 7175675Salfred/* protect list manipulation */ 7275675Salfredstatic struct mtx dev_pager_mtx; 7312820Sphk 741541Srgrimesstruct pagerops devicepagerops = { 75118466Sphk .pgo_init = dev_pager_init, 76118466Sphk .pgo_alloc = dev_pager_alloc, 77118466Sphk .pgo_dealloc = dev_pager_dealloc, 78118466Sphk .pgo_getpages = dev_pager_getpages, 79118466Sphk .pgo_putpages = dev_pager_putpages, 80118466Sphk .pgo_haspage = dev_pager_haspage, 811541Srgrimes}; 821541Srgrimes 83235375Skibstruct pagerops mgtdevicepagerops = { 84235375Skib .pgo_alloc = dev_pager_alloc, 85235375Skib .pgo_dealloc = dev_pager_dealloc, 86235375Skib .pgo_getpages = dev_pager_getpages, 87235375Skib .pgo_putpages = dev_pager_putpages, 88235375Skib .pgo_haspage = dev_pager_haspage, 89235375Skib}; 90235375Skib 91227530Skibstatic int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 92227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color); 93227530Skibstatic void old_dev_pager_dtor(void *handle); 94227530Skibstatic int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 95227530Skib int prot, vm_page_t *mres); 96227530Skib 97227530Skibstatic struct cdev_pager_ops old_dev_pager_ops = { 98227530Skib .cdev_pg_ctor = old_dev_pager_ctor, 99227530Skib .cdev_pg_dtor = old_dev_pager_dtor, 100227530Skib .cdev_pg_fault = old_dev_pager_fault 101227530Skib}; 102227530Skib 10312820Sphkstatic void 1041541Srgrimesdev_pager_init() 1051541Srgrimes{ 1069507Sdg TAILQ_INIT(&dev_pager_object_list); 10793818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 1081541Srgrimes} 1091541Srgrimes 110227530Skibvm_object_t 111227530Skibcdev_pager_lookup(void *handle) 1121541Srgrimes{ 113227530Skib vm_object_t object; 114227530Skib 115227530Skib mtx_lock(&dev_pager_mtx); 116227530Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 117227530Skib mtx_unlock(&dev_pager_mtx); 118227530Skib return (object); 119227530Skib} 120227530Skib 121227530Skibvm_object_t 122227530Skibcdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops, 123227530Skib vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 124227530Skib{ 125171779Skib vm_object_t object, object1; 126124133Salc vm_pindex_t pindex; 127227530Skib u_short color; 1281541Srgrimes 129235375Skib if (tp != OBJT_DEVICE && tp != OBJT_MGTDEVICE) 130227530Skib return (NULL); 131227530Skib 1321541Srgrimes /* 13398630Salc * Offset should be page aligned. 13498630Salc */ 13598630Salc if (foff & PAGE_MASK) 13698630Salc return (NULL); 13798630Salc 13898630Salc size = round_page(size); 139124133Salc pindex = OFF_TO_IDX(foff + size); 14098630Salc 141227530Skib if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0) 142135707Sphk return (NULL); 143171779Skib mtx_lock(&dev_pager_mtx); 1449507Sdg 1459507Sdg /* 1461541Srgrimes * Look up pager, creating as necessary. 1471541Srgrimes */ 148171779Skib object1 = NULL; 1499507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1509507Sdg if (object == NULL) { 1511541Srgrimes /* 152179074Salc * Allocate object and associate it with the pager. Initialize 153179074Salc * the object's pg_color based upon the physical address of the 154179074Salc * device's memory. 1551541Srgrimes */ 156171779Skib mtx_unlock(&dev_pager_mtx); 157227530Skib object1 = vm_object_allocate(tp, pindex); 158179074Salc object1->flags |= OBJ_COLORED; 159227530Skib object1->pg_color = color; 160227530Skib object1->handle = handle; 161227530Skib object1->un_pager.devp.ops = ops; 162245226Sken object1->un_pager.devp.dev = handle; 163224522Skib TAILQ_INIT(&object1->un_pager.devp.devp_pglist); 16475675Salfred mtx_lock(&dev_pager_mtx); 165171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 166171779Skib if (object != NULL) { 167171779Skib /* 168171779Skib * We raced with other thread while allocating object. 169171779Skib */ 170171779Skib if (pindex > object->size) 171171779Skib object->size = pindex; 172171779Skib } else { 173171779Skib object = object1; 174171779Skib object1 = NULL; 175171779Skib object->handle = handle; 176171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 177171779Skib pager_object_list); 178227530Skib KASSERT(object->type == tp, 179227530Skib ("Inconsistent device pager type %p %d", object, tp)); 180171779Skib } 1811541Srgrimes } else { 182124133Salc if (pindex > object->size) 183124133Salc object->size = pindex; 1841541Srgrimes } 185171779Skib mtx_unlock(&dev_pager_mtx); 186224522Skib if (object1 != NULL) { 187224522Skib object1->handle = object1; 188224522Skib mtx_lock(&dev_pager_mtx); 189224522Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object1, 190224522Skib pager_object_list); 191224522Skib mtx_unlock(&dev_pager_mtx); 192224522Skib vm_object_deallocate(object1); 193224522Skib } 1949507Sdg return (object); 1951541Srgrimes} 1961541Srgrimes 197227530Skibstatic vm_object_t 198227530Skibdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 199227530Skib vm_ooffset_t foff, struct ucred *cred) 200227530Skib{ 201227530Skib 202227530Skib return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops, 203227530Skib size, prot, foff, cred)); 204227530Skib} 205227530Skib 206227530Skibvoid 207227530Skibcdev_pager_free_page(vm_object_t object, vm_page_t m) 208227530Skib{ 209227530Skib 210248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 211235375Skib if (object->type == OBJT_MGTDEVICE) { 212235375Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("unmanaged %p", m)); 213235375Skib pmap_remove_all(m); 214235375Skib vm_page_lock(m); 215235375Skib vm_page_remove(m); 216235375Skib vm_page_unlock(m); 217235375Skib } else if (object->type == OBJT_DEVICE) 218235375Skib dev_pager_free_page(object, m); 219235375Skib} 220235375Skib 221235375Skibstatic void 222235375Skibdev_pager_free_page(vm_object_t object, vm_page_t m) 223235375Skib{ 224235375Skib 225248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 226235375Skib KASSERT((object->type == OBJT_DEVICE && 227235375Skib (m->oflags & VPO_UNMANAGED) != 0), 228235375Skib ("Managed device or page obj %p m %p", object, m)); 229254182Skib TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, plinks.q); 230227530Skib vm_page_putfake(m); 231227530Skib} 232227530Skib 23312820Sphkstatic void 2349507Sdgdev_pager_dealloc(object) 2359507Sdg vm_object_t object; 2361541Srgrimes{ 2371541Srgrimes vm_page_t m; 2381541Srgrimes 239248084Sattilio VM_OBJECT_WUNLOCK(object); 240245226Sken object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.dev); 241227530Skib 24275675Salfred mtx_lock(&dev_pager_mtx); 2439507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 24475675Salfred mtx_unlock(&dev_pager_mtx); 245248084Sattilio VM_OBJECT_WLOCK(object); 246235375Skib 247235375Skib if (object->type == OBJT_DEVICE) { 248235375Skib /* 249235375Skib * Free up our fake pages. 250235375Skib */ 251235375Skib while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) 252235375Skib != NULL) 253235375Skib dev_pager_free_page(object, m); 254235375Skib } 255284100Sjhb object->handle = NULL; 256284100Sjhb object->type = OBJT_DEAD; 257227530Skib} 258227530Skib 259227530Skibstatic int 260227530Skibdev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage) 261227530Skib{ 262227530Skib int error, i; 263227530Skib 264248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 265227530Skib error = object->un_pager.devp.ops->cdev_pg_fault(object, 266227530Skib IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]); 267227530Skib 268248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 269227530Skib 270227530Skib for (i = 0; i < count; i++) { 271227530Skib if (i != reqpage) { 272227530Skib vm_page_lock(ma[i]); 273227530Skib vm_page_free(ma[i]); 274227530Skib vm_page_unlock(ma[i]); 275227530Skib } 2761541Srgrimes } 277227530Skib 278227530Skib if (error == VM_PAGER_OK) { 279235375Skib KASSERT((object->type == OBJT_DEVICE && 280235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) != 0) || 281235375Skib (object->type == OBJT_MGTDEVICE && 282235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) == 0), 283235375Skib ("Wrong page type %p %p", ma[reqpage], object)); 284235375Skib if (object->type == OBJT_DEVICE) { 285235375Skib TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, 286254182Skib ma[reqpage], plinks.q); 287235375Skib } 288227530Skib } 289227530Skib 290227530Skib return (error); 2911541Srgrimes} 2921541Srgrimes 29312820Sphkstatic int 294227530Skibold_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, 295227530Skib vm_page_t *mres) 2961541Srgrimes{ 297227530Skib vm_pindex_t pidx; 298112569Sjake vm_paddr_t paddr; 299195649Salc vm_page_t m_paddr, page; 300130585Sphk struct cdev *dev; 301135707Sphk struct cdevsw *csw; 302227530Skib struct file *fpop; 303183383Skib struct thread *td; 304227530Skib vm_memattr_t memattr; 305227530Skib int ref, ret; 3061541Srgrimes 307227530Skib pidx = OFF_TO_IDX(offset); 308195649Salc memattr = object->memattr; 309227530Skib 310248084Sattilio VM_OBJECT_WUNLOCK(object); 311227530Skib 312227530Skib dev = object->handle; 313210923Skib csw = dev_refthread(dev, &ref); 314223823Sattilio if (csw == NULL) { 315248084Sattilio VM_OBJECT_WLOCK(object); 316223823Sattilio return (VM_PAGER_FAIL); 317223823Sattilio } 318183383Skib td = curthread; 319183383Skib fpop = td->td_fpop; 320183383Skib td->td_fpop = NULL; 321227530Skib ret = csw->d_mmap(dev, offset, &paddr, prot, &memattr); 322183383Skib td->td_fpop = fpop; 323210923Skib dev_relthread(dev, ref); 324227530Skib if (ret != 0) { 325227530Skib printf( 326227530Skib "WARNING: dev_pager_getpage: map function returns error %d", ret); 327248084Sattilio VM_OBJECT_WLOCK(object); 328227530Skib return (VM_PAGER_FAIL); 329227530Skib } 330227530Skib 331195649Salc /* If "paddr" is a real page, perform a sanity check on "memattr". */ 332195649Salc if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL && 333195649Salc pmap_page_get_memattr(m_paddr) != memattr) { 334195649Salc memattr = pmap_page_get_memattr(m_paddr); 335195649Salc printf( 336195649Salc "WARNING: A device driver has set \"memattr\" inconsistently.\n"); 337195649Salc } 338227530Skib if (((*mres)->flags & PG_FICTITIOUS) != 0) { 339132884Sdfr /* 340227530Skib * If the passed in result page is a fake page, update it with 341132884Sdfr * the new physical address. 342132884Sdfr */ 343227530Skib page = *mres; 344248084Sattilio VM_OBJECT_WLOCK(object); 345219476Salc vm_page_updatefake(page, paddr, memattr); 346132884Sdfr } else { 347132884Sdfr /* 348132884Sdfr * Replace the passed in reqpage page with our own fake page and 349132884Sdfr * free up the all of the original pages. 350132884Sdfr */ 351219476Salc page = vm_page_getfake(paddr, memattr); 352248084Sattilio VM_OBJECT_WLOCK(object); 353254141Sattilio if (vm_page_replace(page, object, (*mres)->pindex) != *mres) 354254141Sattilio panic("old_dev_pager_fault: invalid page replacement"); 355227530Skib vm_page_lock(*mres); 356227530Skib vm_page_free(*mres); 357227530Skib vm_page_unlock(*mres); 358227530Skib *mres = page; 359132884Sdfr } 360194642Salc page->valid = VM_PAGE_BITS_ALL; 3615455Sdg return (VM_PAGER_OK); 3621541Srgrimes} 3631541Srgrimes 36443129Sdillonstatic void 3659507Sdgdev_pager_putpages(object, m, count, sync, rtvals) 3669507Sdg vm_object_t object; 3679507Sdg vm_page_t *m; 3689507Sdg int count; 3691541Srgrimes boolean_t sync; 3709507Sdg int *rtvals; 3711541Srgrimes{ 372227530Skib 3731541Srgrimes panic("dev_pager_putpage called"); 3741541Srgrimes} 3751541Srgrimes 37612820Sphkstatic boolean_t 37712767Sdysondev_pager_haspage(object, pindex, before, after) 3789507Sdg vm_object_t object; 37912767Sdyson vm_pindex_t pindex; 3809507Sdg int *before; 3819507Sdg int *after; 3821541Srgrimes{ 3839507Sdg if (before != NULL) 3849507Sdg *before = 0; 3859507Sdg if (after != NULL) 3869507Sdg *after = 0; 3875455Sdg return (TRUE); 3881541Srgrimes} 389227530Skib 390227530Skibstatic int 391227530Skibold_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 392227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 393227530Skib{ 394227530Skib struct cdev *dev; 395227530Skib struct cdevsw *csw; 396227530Skib vm_memattr_t dummy; 397227530Skib vm_ooffset_t off; 398227530Skib vm_paddr_t paddr; 399227530Skib unsigned int npages; 400227530Skib int ref; 401227530Skib 402227530Skib /* 403227530Skib * Make sure this device can be mapped. 404227530Skib */ 405227530Skib dev = handle; 406227530Skib csw = dev_refthread(dev, &ref); 407227530Skib if (csw == NULL) 408227530Skib return (ENXIO); 409227530Skib 410227530Skib /* 411227530Skib * Check that the specified range of the device allows the desired 412227530Skib * protection. 413227530Skib * 414227530Skib * XXX assumes VM_PROT_* == PROT_* 415227530Skib */ 416227530Skib npages = OFF_TO_IDX(size); 417263360Skib paddr = 0; /* Make paddr initialized for the case of size == 0. */ 418227530Skib for (off = foff; npages--; off += PAGE_SIZE) { 419227530Skib if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) { 420227530Skib dev_relthread(dev, ref); 421227530Skib return (EINVAL); 422227530Skib } 423227530Skib } 424227530Skib 425227530Skib dev_ref(dev); 426227530Skib dev_relthread(dev, ref); 427227530Skib *color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE); 428227530Skib return (0); 429227530Skib} 430227530Skib 431227530Skibstatic void 432227530Skibold_dev_pager_dtor(void *handle) 433227530Skib{ 434227530Skib 435227530Skib dev_rel(handle); 436227530Skib} 437