1139724Simp/*- 240029Sgibbs * Copyright (c) 1997, 1998 Justin T. Gibbs. 3257230Skib * Copyright (c) 2013 The FreeBSD Foundation 432516Sgibbs * All rights reserved. 532516Sgibbs * 6257230Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7257230Skib * under sponsorship from the FreeBSD Foundation. 8257230Skib * 932516Sgibbs * Redistribution and use in source and binary forms, with or without 1032516Sgibbs * modification, are permitted provided that the following conditions 1132516Sgibbs * are met: 1232516Sgibbs * 1. Redistributions of source code must retain the above copyright 1332516Sgibbs * notice, this list of conditions, and the following disclaimer, 1432516Sgibbs * without modification, immediately at the beginning of the file. 1532516Sgibbs * 2. The name of the author may not be used to endorse or promote products 1632516Sgibbs * derived from this software without specific prior written permission. 1732516Sgibbs * 1832516Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1932516Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2032516Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2132516Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 2232516Sgibbs * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2332516Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2432516Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2532516Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2632516Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2732516Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2832516Sgibbs * SUCH DAMAGE. 2932516Sgibbs */ 3032516Sgibbs 31115683Sobrien#include <sys/cdefs.h> 32115683Sobrien__FBSDID("$FreeBSD$"); 33115683Sobrien 3432516Sgibbs#include <sys/param.h> 3532516Sgibbs#include <sys/systm.h> 3632516Sgibbs#include <sys/malloc.h> 3767551Sjhb#include <sys/bus.h> 38112346Smux#include <sys/kernel.h> 39136805Srwatson#include <sys/ktr.h> 4076827Salfred#include <sys/lock.h> 41246713Skib#include <sys/memdesc.h> 4276827Salfred#include <sys/mutex.h> 43104486Ssam#include <sys/uio.h> 4432516Sgibbs#include <vm/vm.h> 45239008Sjhb#include <vm/vm_extern.h> 46257230Skib#include <vm/pmap.h> 4732516Sgibbs#include <machine/bus.h> 48257230Skib#include <x86/include/busdma_impl.h> 4932516Sgibbs 5095076Salfred/* 51117126Sscottl * Convenience function for manipulating driver locks from busdma (during 52117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 53117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 54117126Sscottl * non-mutex locking scheme don't have to use this at all. 55117126Sscottl */ 56117126Sscottlvoid 57117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 58117126Sscottl{ 59117126Sscottl struct mtx *dmtx; 60117126Sscottl 61117126Sscottl dmtx = (struct mtx *)arg; 62117126Sscottl switch (op) { 63117126Sscottl case BUS_DMA_LOCK: 64117126Sscottl mtx_lock(dmtx); 65117126Sscottl break; 66117126Sscottl case BUS_DMA_UNLOCK: 67117126Sscottl mtx_unlock(dmtx); 68117126Sscottl break; 69117126Sscottl default: 70117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 71117126Sscottl } 72117126Sscottl} 73117126Sscottl 74117126Sscottl/* 75117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 76117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 77117126Sscottl * with the tag are meant to never be defered. 78117126Sscottl * XXX Should have a way to identify which driver is responsible here. 79117126Sscottl */ 80257230Skibvoid 81257230Skibbus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op) 82117126Sscottl{ 83257230Skib 84117126Sscottl panic("driver error: busdma dflt_lock called"); 85117126Sscottl} 86117126Sscottl 8732516Sgibbs/* 88257230Skib * Return true if a match is made. 89257230Skib * 90257230Skib * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 91257230Skib * 92257230Skib * If paddr is within the bounds of the dma tag then call the filter callback 93257230Skib * to check for a match, if there is no filter callback then assume a match. 9432516Sgibbs */ 9532516Sgibbsint 96257230Skibbus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) 9732516Sgibbs{ 98257230Skib int retval; 9932516Sgibbs 100257230Skib retval = 0; 101257230Skib do { 102257230Skib if (((paddr > tc->lowaddr && paddr <= tc->highaddr) || 103257230Skib ((paddr & (tc->alignment - 1)) != 0)) && 104257230Skib (tc->filter == NULL || 105257230Skib (*tc->filter)(tc->filterarg, paddr) != 0)) 106257230Skib retval = 1; 107257230Skib 108257230Skib tc = tc->parent; 109257230Skib } while (retval == 0 && tc != NULL); 110257230Skib return (retval); 111257230Skib} 112257230Skib 113257230Skibint 114257230Skibcommon_bus_dma_tag_create(struct bus_dma_tag_common *parent, 115257230Skib bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, 116257230Skib bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, 117257230Skib bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 118257230Skib bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) 119257230Skib{ 120257230Skib void *newtag; 121257230Skib struct bus_dma_tag_common *common; 122257230Skib 123257230Skib KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); 124131529Sscottl /* Basic sanity checking */ 125131529Sscottl if (boundary != 0 && boundary < maxsegsz) 126131529Sscottl maxsegsz = boundary; 127257230Skib if (maxsegsz == 0) 128170564Smjacob return (EINVAL); 12932516Sgibbs /* Return a NULL tag on failure */ 13032516Sgibbs *dmat = NULL; 13132516Sgibbs 132257230Skib newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); 133136805Srwatson if (newtag == NULL) { 134143293Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 135248968Skib __func__, newtag, 0, ENOMEM); 13632516Sgibbs return (ENOMEM); 137136805Srwatson } 13832516Sgibbs 139257230Skib common = newtag; 140257230Skib common->impl = &bus_dma_bounce_impl; 141257230Skib common->parent = parent; 142257230Skib common->alignment = alignment; 143257230Skib common->boundary = boundary; 144257230Skib common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 145257230Skib common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 146257230Skib common->filter = filter; 147257230Skib common->filterarg = filterarg; 148257230Skib common->maxsize = maxsize; 149257230Skib common->nsegments = nsegments; 150257230Skib common->maxsegsz = maxsegsz; 151257230Skib common->flags = flags; 152257230Skib common->ref_count = 1; /* Count ourself */ 153117126Sscottl if (lockfunc != NULL) { 154257230Skib common->lockfunc = lockfunc; 155257230Skib common->lockfuncarg = lockfuncarg; 156117126Sscottl } else { 157257230Skib common->lockfunc = bus_dma_dflt_lock; 158257230Skib common->lockfuncarg = NULL; 159117126Sscottl } 160118246Sscottl 16132516Sgibbs /* Take into account any restrictions imposed by our parent tag */ 16232516Sgibbs if (parent != NULL) { 163257230Skib common->impl = parent->impl; 164257230Skib common->lowaddr = MIN(parent->lowaddr, common->lowaddr); 165257230Skib common->highaddr = MAX(parent->highaddr, common->highaddr); 166257230Skib if (common->boundary == 0) 167257230Skib common->boundary = parent->boundary; 168257230Skib else if (parent->boundary != 0) { 169257230Skib common->boundary = MIN(parent->boundary, 170257230Skib common->boundary); 171257230Skib } 172257230Skib if (common->filter == NULL) { 17332516Sgibbs /* 17432516Sgibbs * Short circuit looking at our parent directly 17535256Sdes * since we have encapsulated all of its information 17632516Sgibbs */ 177257230Skib common->filter = parent->filter; 178257230Skib common->filterarg = parent->filterarg; 179257230Skib common->parent = parent->parent; 18032516Sgibbs } 181257230Skib atomic_add_int(&parent->ref_count, 1); 18232516Sgibbs } 183257230Skib *dmat = common; 184257230Skib return (0); 185257230Skib} 186137965Sscottl 187257230Skib/* 188257230Skib * Allocate a device specific dma_tag. 189257230Skib */ 190257230Skibint 191257230Skibbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 192257230Skib bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 193257230Skib bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 194257230Skib int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 195257230Skib void *lockfuncarg, bus_dma_tag_t *dmat) 196257230Skib{ 197257230Skib struct bus_dma_tag_common *tc; 198257230Skib int error; 199137965Sscottl 200257230Skib if (parent == NULL) { 201257230Skib error = bus_dma_bounce_impl.tag_create(parent, alignment, 202257230Skib boundary, lowaddr, highaddr, filter, filterarg, maxsize, 203257230Skib nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 20432516Sgibbs } else { 205257230Skib tc = (struct bus_dma_tag_common *)parent; 206257230Skib error = tc->impl->tag_create(parent, alignment, 207257230Skib boundary, lowaddr, highaddr, filter, filterarg, maxsize, 208257230Skib nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 20932516Sgibbs } 21032516Sgibbs return (error); 21132516Sgibbs} 21232516Sgibbs 21332516Sgibbsint 21432516Sgibbsbus_dma_tag_destroy(bus_dma_tag_t dmat) 21532516Sgibbs{ 216257230Skib struct bus_dma_tag_common *tc; 217136805Srwatson 218257230Skib tc = (struct bus_dma_tag_common *)dmat; 219257230Skib return (tc->impl->tag_destroy(dmat)); 22032516Sgibbs} 22132516Sgibbs 22232516Sgibbs/* 22332516Sgibbs * Allocate a handle for mapping from kva/uva/physical 22432516Sgibbs * address space into bus device space. 22532516Sgibbs */ 22632516Sgibbsint 22732516Sgibbsbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 22832516Sgibbs{ 229257230Skib struct bus_dma_tag_common *tc; 23032516Sgibbs 231257230Skib tc = (struct bus_dma_tag_common *)dmat; 232257230Skib return (tc->impl->map_create(dmat, flags, mapp)); 23332516Sgibbs} 23432516Sgibbs 23532516Sgibbs/* 23632516Sgibbs * Destroy a handle for mapping from kva/uva/physical 23732516Sgibbs * address space into bus device space. 23832516Sgibbs */ 23932516Sgibbsint 24032516Sgibbsbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 24132516Sgibbs{ 242257230Skib struct bus_dma_tag_common *tc; 243257230Skib 244257230Skib tc = (struct bus_dma_tag_common *)dmat; 245257230Skib return (tc->impl->map_destroy(dmat, map)); 24632516Sgibbs} 24732516Sgibbs 24835767Sgibbs 24935767Sgibbs/* 25035767Sgibbs * Allocate a piece of memory that can be efficiently mapped into 25135767Sgibbs * bus device space based on the constraints lited in the dma tag. 25235767Sgibbs * A dmamap to for use with dmamap_load is also allocated. 25335767Sgibbs */ 25435767Sgibbsint 255115316Sscottlbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 256257230Skib bus_dmamap_t *mapp) 25735767Sgibbs{ 258257230Skib struct bus_dma_tag_common *tc; 259118081Smux 260257230Skib tc = (struct bus_dma_tag_common *)dmat; 261257230Skib return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp)); 26235767Sgibbs} 26335767Sgibbs 26435767Sgibbs/* 26535767Sgibbs * Free a piece of memory and it's allociated dmamap, that was allocated 26695076Salfred * via bus_dmamem_alloc. Make the same choice for free/contigfree. 26735767Sgibbs */ 26835767Sgibbsvoid 269115316Sscottlbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 27035767Sgibbs{ 271257230Skib struct bus_dma_tag_common *tc; 27235767Sgibbs 273257230Skib tc = (struct bus_dma_tag_common *)dmat; 274257230Skib tc->impl->mem_free(dmat, vaddr, map); 275246713Skib} 276246713Skib 277162211Sscottl/* 278257230Skib * Utility function to load a physical buffer. segp contains 279257230Skib * the starting segment on entrace, and the ending segment on exit. 280246713Skib */ 281257230Skibint 282257230Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 283257230Skib bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 284246713Skib{ 285257230Skib struct bus_dma_tag_common *tc; 286246713Skib 287257230Skib tc = (struct bus_dma_tag_common *)dmat; 288257230Skib return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs, 289257230Skib segp)); 290246713Skib} 291246713Skib 292246713Skibint 293257230Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, 294257230Skib bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, 295257230Skib int *segp) 296246713Skib{ 297257230Skib struct bus_dma_tag_common *tc; 298246713Skib 299257230Skib tc = (struct bus_dma_tag_common *)dmat; 300257230Skib return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags, 301257230Skib segs, segp)); 302246713Skib} 303246713Skib 304246713Skib/* 305246713Skib * Utility function to load a linear buffer. segp contains 306246713Skib * the starting segment on entrace, and the ending segment on exit. 307246713Skib */ 308246713Skibint 309257230Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 310257230Skib bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 311257230Skib int *segp) 312162211Sscottl{ 313257230Skib struct bus_dma_tag_common *tc; 314162211Sscottl 315257230Skib tc = (struct bus_dma_tag_common *)dmat; 316257230Skib return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs, 317257230Skib segp)); 318104486Ssam} 319104486Ssam 320246713Skibvoid 321246713Skib__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 322257230Skib struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 323113459Ssimokawa{ 324257230Skib struct bus_dma_tag_common *tc; 325257230Skib 326257230Skib tc = (struct bus_dma_tag_common *)dmat; 327257230Skib tc->impl->map_waitok(dmat, map, mem, callback, callback_arg); 328113459Ssimokawa} 329113459Ssimokawa 330246713Skibbus_dma_segment_t * 331246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 332257230Skib bus_dma_segment_t *segs, int nsegs, int error) 333104486Ssam{ 334257230Skib struct bus_dma_tag_common *tc; 335104486Ssam 336257230Skib tc = (struct bus_dma_tag_common *)dmat; 337257230Skib return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); 338162275Sscottl} 339162275Sscottl 340104486Ssam/* 34132516Sgibbs * Release the mapping held by map. 34232516Sgibbs */ 34332516Sgibbsvoid 34432516Sgibbs_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 34532516Sgibbs{ 346257230Skib struct bus_dma_tag_common *tc; 34732516Sgibbs 348257230Skib tc = (struct bus_dma_tag_common *)dmat; 349257230Skib tc->impl->map_unload(dmat, map); 35032516Sgibbs} 35132516Sgibbs 35232516Sgibbsvoid 353115343Sscottl_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 35432516Sgibbs{ 355257230Skib struct bus_dma_tag_common *tc; 35632516Sgibbs 357257230Skib tc = (struct bus_dma_tag_common *)dmat; 358257230Skib tc->impl->map_sync(dmat, map, op); 35932516Sgibbs} 360