1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: stable/10/sys/vm/device_pager.c 320439 2017-06-28 06:13:58Z alc $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 47248084Sattilio#include <sys/rwlock.h> 4875675Salfred#include <sys/sx.h> 491541Srgrimes 501541Srgrimes#include <vm/vm.h> 51239065Skib#include <vm/vm_param.h> 5212662Sdg#include <vm/vm_object.h> 531541Srgrimes#include <vm/vm_page.h> 549507Sdg#include <vm/vm_pager.h> 55243132Skib#include <vm/vm_phys.h> 5692748Sjeff#include <vm/uma.h> 571541Srgrimes 5892727Salfredstatic void dev_pager_init(void); 5992727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 60194766Skib vm_ooffset_t, struct ucred *); 6192727Salfredstatic void dev_pager_dealloc(vm_object_t); 6292727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 63291905Skibstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, int, int *); 64291905Skibstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 65235375Skibstatic void dev_pager_free_page(vm_object_t object, vm_page_t m); 661541Srgrimes 6712820Sphk/* list of device pager objects */ 6812820Sphkstatic struct pagerlst dev_pager_object_list; 6975675Salfred/* protect list manipulation */ 7075675Salfredstatic struct mtx dev_pager_mtx; 7112820Sphk 721541Srgrimesstruct pagerops devicepagerops = { 73118466Sphk .pgo_init = dev_pager_init, 74118466Sphk .pgo_alloc = dev_pager_alloc, 75118466Sphk .pgo_dealloc = dev_pager_dealloc, 76118466Sphk .pgo_getpages = dev_pager_getpages, 77118466Sphk .pgo_putpages = dev_pager_putpages, 78118466Sphk .pgo_haspage = dev_pager_haspage, 791541Srgrimes}; 801541Srgrimes 81235375Skibstruct pagerops mgtdevicepagerops = { 82235375Skib .pgo_alloc = dev_pager_alloc, 83235375Skib .pgo_dealloc = dev_pager_dealloc, 84235375Skib .pgo_getpages = dev_pager_getpages, 85235375Skib .pgo_putpages = dev_pager_putpages, 86235375Skib .pgo_haspage = dev_pager_haspage, 87235375Skib}; 88235375Skib 89227530Skibstatic int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 90227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color); 91227530Skibstatic void old_dev_pager_dtor(void *handle); 92227530Skibstatic int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 93227530Skib int prot, vm_page_t *mres); 94227530Skib 95227530Skibstatic struct cdev_pager_ops old_dev_pager_ops = { 96227530Skib .cdev_pg_ctor = old_dev_pager_ctor, 97227530Skib .cdev_pg_dtor = old_dev_pager_dtor, 98227530Skib .cdev_pg_fault = old_dev_pager_fault 99227530Skib}; 100227530Skib 10112820Sphkstatic void 102291905Skibdev_pager_init(void) 1031541Srgrimes{ 104291905Skib 1059507Sdg TAILQ_INIT(&dev_pager_object_list); 10693818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 1071541Srgrimes} 1081541Srgrimes 109227530Skibvm_object_t 110227530Skibcdev_pager_lookup(void *handle) 1111541Srgrimes{ 112227530Skib vm_object_t object; 113227530Skib 114227530Skib mtx_lock(&dev_pager_mtx); 115227530Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 116227530Skib mtx_unlock(&dev_pager_mtx); 117227530Skib return (object); 118227530Skib} 119227530Skib 120227530Skibvm_object_t 121227530Skibcdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops, 122227530Skib vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 123227530Skib{ 124171779Skib vm_object_t object, object1; 125124133Salc vm_pindex_t pindex; 126227530Skib u_short color; 1271541Srgrimes 128235375Skib if (tp != OBJT_DEVICE && tp != OBJT_MGTDEVICE) 129227530Skib return (NULL); 130227530Skib 1311541Srgrimes /* 13298630Salc * Offset should be page aligned. 13398630Salc */ 13498630Salc if (foff & PAGE_MASK) 13598630Salc return (NULL); 13698630Salc 13798630Salc size = round_page(size); 138124133Salc pindex = OFF_TO_IDX(foff + size); 13998630Salc 140227530Skib if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0) 141135707Sphk return (NULL); 142171779Skib mtx_lock(&dev_pager_mtx); 1439507Sdg 1449507Sdg /* 1451541Srgrimes * Look up pager, creating as necessary. 1461541Srgrimes */ 147171779Skib object1 = NULL; 1489507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1499507Sdg if (object == NULL) { 1501541Srgrimes /* 151179074Salc * Allocate object and associate it with the pager. Initialize 152179074Salc * the object's pg_color based upon the physical address of the 153179074Salc * device's memory. 1541541Srgrimes */ 155171779Skib mtx_unlock(&dev_pager_mtx); 156227530Skib object1 = vm_object_allocate(tp, pindex); 157179074Salc object1->flags |= OBJ_COLORED; 158227530Skib object1->pg_color = color; 159227530Skib object1->handle = handle; 160227530Skib object1->un_pager.devp.ops = ops; 161245226Sken object1->un_pager.devp.dev = handle; 162224522Skib TAILQ_INIT(&object1->un_pager.devp.devp_pglist); 16375675Salfred mtx_lock(&dev_pager_mtx); 164171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 165171779Skib if (object != NULL) { 166171779Skib /* 167171779Skib * We raced with other thread while allocating object. 168171779Skib */ 169171779Skib if (pindex > object->size) 170171779Skib object->size = pindex; 171171779Skib } else { 172171779Skib object = object1; 173171779Skib object1 = NULL; 174171779Skib object->handle = handle; 175171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 176171779Skib pager_object_list); 177227530Skib KASSERT(object->type == tp, 178227530Skib ("Inconsistent device pager type %p %d", object, tp)); 179171779Skib } 1801541Srgrimes } else { 181124133Salc if (pindex > object->size) 182124133Salc object->size = pindex; 1831541Srgrimes } 184171779Skib mtx_unlock(&dev_pager_mtx); 185224522Skib if (object1 != NULL) { 186224522Skib object1->handle = object1; 187224522Skib mtx_lock(&dev_pager_mtx); 188224522Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object1, 189224522Skib pager_object_list); 190224522Skib mtx_unlock(&dev_pager_mtx); 191224522Skib vm_object_deallocate(object1); 192224522Skib } 1939507Sdg return (object); 1941541Srgrimes} 1951541Srgrimes 196227530Skibstatic vm_object_t 197227530Skibdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 198227530Skib vm_ooffset_t foff, struct ucred *cred) 199227530Skib{ 200227530Skib 201227530Skib return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops, 202227530Skib size, prot, foff, cred)); 203227530Skib} 204227530Skib 205227530Skibvoid 206227530Skibcdev_pager_free_page(vm_object_t object, vm_page_t m) 207227530Skib{ 208227530Skib 209248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 210235375Skib if (object->type == OBJT_MGTDEVICE) { 211235375Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("unmanaged %p", m)); 212235375Skib pmap_remove_all(m); 213235375Skib vm_page_lock(m); 214235375Skib vm_page_remove(m); 215235375Skib vm_page_unlock(m); 216235375Skib } else if (object->type == OBJT_DEVICE) 217235375Skib dev_pager_free_page(object, m); 218235375Skib} 219235375Skib 220235375Skibstatic void 221235375Skibdev_pager_free_page(vm_object_t object, vm_page_t m) 222235375Skib{ 223235375Skib 224248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 225235375Skib KASSERT((object->type == OBJT_DEVICE && 226235375Skib (m->oflags & VPO_UNMANAGED) != 0), 227235375Skib ("Managed device or page obj %p m %p", object, m)); 228254182Skib TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, plinks.q); 229227530Skib vm_page_putfake(m); 230227530Skib} 231227530Skib 23212820Sphkstatic void 233291905Skibdev_pager_dealloc(vm_object_t object) 2341541Srgrimes{ 2351541Srgrimes vm_page_t m; 2361541Srgrimes 237248084Sattilio VM_OBJECT_WUNLOCK(object); 238245226Sken object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.dev); 239227530Skib 24075675Salfred mtx_lock(&dev_pager_mtx); 2419507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 24275675Salfred mtx_unlock(&dev_pager_mtx); 243248084Sattilio VM_OBJECT_WLOCK(object); 244235375Skib 245235375Skib if (object->type == OBJT_DEVICE) { 246235375Skib /* 247235375Skib * Free up our fake pages. 248235375Skib */ 249235375Skib while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) 250235375Skib != NULL) 251235375Skib dev_pager_free_page(object, m); 252235375Skib } 253284100Sjhb object->handle = NULL; 254284100Sjhb object->type = OBJT_DEAD; 255227530Skib} 256227530Skib 257227530Skibstatic int 258227530Skibdev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage) 259227530Skib{ 260227530Skib int error, i; 261227530Skib 262248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 263227530Skib error = object->un_pager.devp.ops->cdev_pg_fault(object, 264227530Skib IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]); 265227530Skib 266248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 267227530Skib 268227530Skib for (i = 0; i < count; i++) { 269227530Skib if (i != reqpage) { 270227530Skib vm_page_lock(ma[i]); 271227530Skib vm_page_free(ma[i]); 272227530Skib vm_page_unlock(ma[i]); 273227530Skib } 2741541Srgrimes } 275227530Skib 276227530Skib if (error == VM_PAGER_OK) { 277235375Skib KASSERT((object->type == OBJT_DEVICE && 278235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) != 0) || 279235375Skib (object->type == OBJT_MGTDEVICE && 280235375Skib (ma[reqpage]->oflags & VPO_UNMANAGED) == 0), 281235375Skib ("Wrong page type %p %p", ma[reqpage], object)); 282235375Skib if (object->type == OBJT_DEVICE) { 283235375Skib TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, 284254182Skib ma[reqpage], plinks.q); 285235375Skib } 286227530Skib } 287227530Skib 288227530Skib return (error); 2891541Srgrimes} 2901541Srgrimes 29112820Sphkstatic int 292227530Skibold_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, 293227530Skib vm_page_t *mres) 2941541Srgrimes{ 295112569Sjake vm_paddr_t paddr; 296195649Salc vm_page_t m_paddr, page; 297130585Sphk struct cdev *dev; 298135707Sphk struct cdevsw *csw; 299227530Skib struct file *fpop; 300183383Skib struct thread *td; 301299231Skib vm_memattr_t memattr, memattr1; 302227530Skib int ref, ret; 3031541Srgrimes 304195649Salc memattr = object->memattr; 305227530Skib 306248084Sattilio VM_OBJECT_WUNLOCK(object); 307227530Skib 308227530Skib dev = object->handle; 309210923Skib csw = dev_refthread(dev, &ref); 310223823Sattilio if (csw == NULL) { 311248084Sattilio VM_OBJECT_WLOCK(object); 312223823Sattilio return (VM_PAGER_FAIL); 313223823Sattilio } 314183383Skib td = curthread; 315183383Skib fpop = td->td_fpop; 316183383Skib td->td_fpop = NULL; 317227530Skib ret = csw->d_mmap(dev, offset, &paddr, prot, &memattr); 318183383Skib td->td_fpop = fpop; 319210923Skib dev_relthread(dev, ref); 320227530Skib if (ret != 0) { 321227530Skib printf( 322227530Skib "WARNING: dev_pager_getpage: map function returns error %d", ret); 323248084Sattilio VM_OBJECT_WLOCK(object); 324227530Skib return (VM_PAGER_FAIL); 325227530Skib } 326227530Skib 327195649Salc /* If "paddr" is a real page, perform a sanity check on "memattr". */ 328195649Salc if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL && 329299231Skib (memattr1 = pmap_page_get_memattr(m_paddr)) != memattr) { 330299231Skib /* 331299231Skib * For the /dev/mem d_mmap routine to return the 332299231Skib * correct memattr, pmap_page_get_memattr() needs to 333299231Skib * be called, which we do there. 334299231Skib */ 335299231Skib if ((csw->d_flags & D_MEM) == 0) { 336299231Skib printf("WARNING: Device driver %s has set " 337299231Skib "\"memattr\" inconsistently (drv %u pmap %u).\n", 338299231Skib csw->d_name, memattr, memattr1); 339299231Skib } 340299231Skib memattr = memattr1; 341195649Salc } 342227530Skib if (((*mres)->flags & PG_FICTITIOUS) != 0) { 343132884Sdfr /* 344227530Skib * If the passed in result page is a fake page, update it with 345132884Sdfr * the new physical address. 346132884Sdfr */ 347227530Skib page = *mres; 348248084Sattilio VM_OBJECT_WLOCK(object); 349219476Salc vm_page_updatefake(page, paddr, memattr); 350132884Sdfr } else { 351132884Sdfr /* 352132884Sdfr * Replace the passed in reqpage page with our own fake page and 353132884Sdfr * free up the all of the original pages. 354132884Sdfr */ 355219476Salc page = vm_page_getfake(paddr, memattr); 356248084Sattilio VM_OBJECT_WLOCK(object); 357254141Sattilio if (vm_page_replace(page, object, (*mres)->pindex) != *mres) 358254141Sattilio panic("old_dev_pager_fault: invalid page replacement"); 359227530Skib vm_page_lock(*mres); 360227530Skib vm_page_free(*mres); 361227530Skib vm_page_unlock(*mres); 362227530Skib *mres = page; 363132884Sdfr } 364194642Salc page->valid = VM_PAGE_BITS_ALL; 3655455Sdg return (VM_PAGER_OK); 3661541Srgrimes} 3671541Srgrimes 36843129Sdillonstatic void 369291905Skibdev_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, 370291905Skib int *rtvals) 3711541Srgrimes{ 372227530Skib 3731541Srgrimes panic("dev_pager_putpage called"); 3741541Srgrimes} 3751541Srgrimes 37612820Sphkstatic boolean_t 377291905Skibdev_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 378291905Skib int *after) 3791541Srgrimes{ 380291905Skib 3819507Sdg if (before != NULL) 3829507Sdg *before = 0; 3839507Sdg if (after != NULL) 3849507Sdg *after = 0; 3855455Sdg return (TRUE); 3861541Srgrimes} 387227530Skib 388227530Skibstatic int 389227530Skibold_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 390227530Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 391227530Skib{ 392227530Skib struct cdev *dev; 393227530Skib struct cdevsw *csw; 394227530Skib vm_memattr_t dummy; 395227530Skib vm_ooffset_t off; 396227530Skib vm_paddr_t paddr; 397227530Skib unsigned int npages; 398227530Skib int ref; 399227530Skib 400227530Skib /* 401227530Skib * Make sure this device can be mapped. 402227530Skib */ 403227530Skib dev = handle; 404227530Skib csw = dev_refthread(dev, &ref); 405227530Skib if (csw == NULL) 406227530Skib return (ENXIO); 407227530Skib 408227530Skib /* 409227530Skib * Check that the specified range of the device allows the desired 410227530Skib * protection. 411227530Skib * 412227530Skib * XXX assumes VM_PROT_* == PROT_* 413227530Skib */ 414227530Skib npages = OFF_TO_IDX(size); 415263360Skib paddr = 0; /* Make paddr initialized for the case of size == 0. */ 416227530Skib for (off = foff; npages--; off += PAGE_SIZE) { 417227530Skib if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) { 418227530Skib dev_relthread(dev, ref); 419227530Skib return (EINVAL); 420227530Skib } 421227530Skib } 422227530Skib 423227530Skib dev_ref(dev); 424227530Skib dev_relthread(dev, ref); 425227530Skib *color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE); 426227530Skib return (0); 427227530Skib} 428227530Skib 429227530Skibstatic void 430227530Skibold_dev_pager_dtor(void *handle) 431227530Skib{ 432227530Skib 433227530Skib dev_rel(handle); 434227530Skib} 435