1139724Simp/*- 240029Sgibbs * Copyright (c) 1997, 1998 Justin T. Gibbs. 3259511Skib * Copyright (c) 2013 The FreeBSD Foundation 432516Sgibbs * All rights reserved. 532516Sgibbs * 6259511Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7259511Skib * under sponsorship from the FreeBSD Foundation. 8259511Skib * 932516Sgibbs * Redistribution and use in source and binary forms, with or without 1032516Sgibbs * modification, are permitted provided that the following conditions 1132516Sgibbs * are met: 1232516Sgibbs * 1. Redistributions of source code must retain the above copyright 1332516Sgibbs * notice, this list of conditions, and the following disclaimer, 1432516Sgibbs * without modification, immediately at the beginning of the file. 1532516Sgibbs * 2. The name of the author may not be used to endorse or promote products 1632516Sgibbs * derived from this software without specific prior written permission. 1732516Sgibbs * 1832516Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1932516Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2032516Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2132516Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 2232516Sgibbs * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2332516Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2432516Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2532516Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2632516Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2732516Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2832516Sgibbs * SUCH DAMAGE. 2932516Sgibbs */ 3032516Sgibbs 31115683Sobrien#include <sys/cdefs.h> 32115683Sobrien__FBSDID("$FreeBSD$"); 33115683Sobrien 3432516Sgibbs#include <sys/param.h> 3532516Sgibbs#include <sys/systm.h> 3632516Sgibbs#include <sys/malloc.h> 3767551Sjhb#include <sys/bus.h> 38112346Smux#include <sys/kernel.h> 39136805Srwatson#include <sys/ktr.h> 4076827Salfred#include <sys/lock.h> 41246713Skib#include <sys/memdesc.h> 4276827Salfred#include <sys/mutex.h> 43104486Ssam#include <sys/uio.h> 4432516Sgibbs#include <vm/vm.h> 45239008Sjhb#include <vm/vm_extern.h> 46259511Skib#include <vm/pmap.h> 4732516Sgibbs#include <machine/bus.h> 48259511Skib#include <x86/include/busdma_impl.h> 4932516Sgibbs 5095076Salfred/* 51117126Sscottl * Convenience function for manipulating driver locks from busdma (during 52117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 53117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 54117126Sscottl * non-mutex locking scheme don't have to use this at all. 55117126Sscottl */ 56117126Sscottlvoid 57117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 58117126Sscottl{ 59117126Sscottl struct mtx *dmtx; 60117126Sscottl 61117126Sscottl dmtx = (struct mtx *)arg; 62117126Sscottl switch (op) { 63117126Sscottl case BUS_DMA_LOCK: 64117126Sscottl mtx_lock(dmtx); 65117126Sscottl break; 66117126Sscottl case BUS_DMA_UNLOCK: 67117126Sscottl mtx_unlock(dmtx); 68117126Sscottl break; 69117126Sscottl default: 70117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 71117126Sscottl } 72117126Sscottl} 73117126Sscottl 74117126Sscottl/* 75117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 76117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 77117126Sscottl * with the tag are meant to never be defered. 78117126Sscottl * XXX Should have a way to identify which driver is responsible here. 79117126Sscottl */ 80259511Skibvoid 81259511Skibbus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op) 82117126Sscottl{ 83259511Skib 84117126Sscottl panic("driver error: busdma dflt_lock called"); 85117126Sscottl} 86117126Sscottl 8732516Sgibbs/* 88259511Skib * Return true if a match is made. 89259511Skib * 90259511Skib * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 91259511Skib * 92259511Skib * If paddr is within the bounds of the dma tag then call the filter callback 93259511Skib * to check for a match, if there is no filter callback then assume a match. 9432516Sgibbs */ 9532516Sgibbsint 96259511Skibbus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) 9732516Sgibbs{ 98259511Skib int retval; 9932516Sgibbs 100259511Skib retval = 0; 101259511Skib do { 102259511Skib if (((paddr > tc->lowaddr && paddr <= tc->highaddr) || 103259511Skib ((paddr & (tc->alignment - 1)) != 0)) && 104259511Skib (tc->filter == NULL || 105259511Skib (*tc->filter)(tc->filterarg, paddr) != 0)) 106259511Skib retval = 1; 107259511Skib 108259511Skib tc = tc->parent; 109259511Skib } while (retval == 0 && tc != NULL); 110259511Skib return (retval); 111259511Skib} 112259511Skib 113259511Skibint 114259511Skibcommon_bus_dma_tag_create(struct bus_dma_tag_common *parent, 115259511Skib bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, 116259511Skib bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, 117259511Skib bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 118259511Skib bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) 119259511Skib{ 120259511Skib void *newtag; 121259511Skib struct bus_dma_tag_common *common; 122259511Skib 123259511Skib KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); 124131529Sscottl /* Basic sanity checking */ 125131529Sscottl if (boundary != 0 && boundary < maxsegsz) 126131529Sscottl maxsegsz = boundary; 127259511Skib if (maxsegsz == 0) 128170564Smjacob return (EINVAL); 12932516Sgibbs /* Return a NULL tag on failure */ 13032516Sgibbs *dmat = NULL; 13132516Sgibbs 132259511Skib newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); 133136805Srwatson if (newtag == NULL) { 134143293Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 135248968Skib __func__, newtag, 0, ENOMEM); 13632516Sgibbs return (ENOMEM); 137136805Srwatson } 13832516Sgibbs 139259511Skib common = newtag; 140259511Skib common->impl = &bus_dma_bounce_impl; 141259511Skib common->parent = parent; 142259511Skib common->alignment = alignment; 143259511Skib common->boundary = boundary; 144259511Skib common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 145259511Skib common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 146259511Skib common->filter = filter; 147259511Skib common->filterarg = filterarg; 148259511Skib common->maxsize = maxsize; 149259511Skib common->nsegments = nsegments; 150259511Skib common->maxsegsz = maxsegsz; 151259511Skib common->flags = flags; 152259511Skib common->ref_count = 1; /* Count ourself */ 153117126Sscottl if (lockfunc != NULL) { 154259511Skib common->lockfunc = lockfunc; 155259511Skib common->lockfuncarg = lockfuncarg; 156117126Sscottl } else { 157259511Skib common->lockfunc = bus_dma_dflt_lock; 158259511Skib common->lockfuncarg = NULL; 159117126Sscottl } 160118246Sscottl 16132516Sgibbs /* Take into account any restrictions imposed by our parent tag */ 16232516Sgibbs if (parent != NULL) { 163259511Skib common->impl = parent->impl; 164259511Skib common->lowaddr = MIN(parent->lowaddr, common->lowaddr); 165259511Skib common->highaddr = MAX(parent->highaddr, common->highaddr); 166259511Skib if (common->boundary == 0) 167259511Skib common->boundary = parent->boundary; 168259511Skib else if (parent->boundary != 0) { 169259511Skib common->boundary = MIN(parent->boundary, 170259511Skib common->boundary); 171259511Skib } 172259511Skib if (common->filter == NULL) { 17332516Sgibbs /* 17432516Sgibbs * Short circuit looking at our parent directly 17535256Sdes * since we have encapsulated all of its information 17632516Sgibbs */ 177259511Skib common->filter = parent->filter; 178259511Skib common->filterarg = parent->filterarg; 179259511Skib common->parent = parent->parent; 18032516Sgibbs } 181259511Skib atomic_add_int(&parent->ref_count, 1); 18232516Sgibbs } 183259511Skib *dmat = common; 184259511Skib return (0); 185259511Skib} 186137965Sscottl 187259511Skib/* 188259511Skib * Allocate a device specific dma_tag. 189259511Skib */ 190259511Skibint 191259511Skibbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 192259511Skib bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 193259511Skib bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 194259511Skib int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 195259511Skib void *lockfuncarg, bus_dma_tag_t *dmat) 196259511Skib{ 197259511Skib struct bus_dma_tag_common *tc; 198259511Skib int error; 199137965Sscottl 200259511Skib if (parent == NULL) { 201259511Skib error = bus_dma_bounce_impl.tag_create(parent, alignment, 202259511Skib boundary, lowaddr, highaddr, filter, filterarg, maxsize, 203259511Skib nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 20432516Sgibbs } else { 205259511Skib tc = (struct bus_dma_tag_common *)parent; 206259511Skib error = tc->impl->tag_create(parent, alignment, 207259511Skib boundary, lowaddr, highaddr, filter, filterarg, maxsize, 208259511Skib nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 20932516Sgibbs } 21032516Sgibbs return (error); 21132516Sgibbs} 21232516Sgibbs 21332516Sgibbsint 21432516Sgibbsbus_dma_tag_destroy(bus_dma_tag_t dmat) 21532516Sgibbs{ 216259511Skib struct bus_dma_tag_common *tc; 217136805Srwatson 218259511Skib tc = (struct bus_dma_tag_common *)dmat; 219259511Skib return (tc->impl->tag_destroy(dmat)); 22032516Sgibbs} 22132516Sgibbs 22232516Sgibbs/* 22332516Sgibbs * Allocate a handle for mapping from kva/uva/physical 22432516Sgibbs * address space into bus device space. 22532516Sgibbs */ 22632516Sgibbsint 22732516Sgibbsbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 22832516Sgibbs{ 229259511Skib struct bus_dma_tag_common *tc; 23032516Sgibbs 231259511Skib tc = (struct bus_dma_tag_common *)dmat; 232259511Skib return (tc->impl->map_create(dmat, flags, mapp)); 23332516Sgibbs} 23432516Sgibbs 23532516Sgibbs/* 23632516Sgibbs * Destroy a handle for mapping from kva/uva/physical 23732516Sgibbs * address space into bus device space. 23832516Sgibbs */ 23932516Sgibbsint 24032516Sgibbsbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 24132516Sgibbs{ 242259511Skib struct bus_dma_tag_common *tc; 243259511Skib 244259511Skib tc = (struct bus_dma_tag_common *)dmat; 245259511Skib return (tc->impl->map_destroy(dmat, map)); 24632516Sgibbs} 24732516Sgibbs 24835767Sgibbs 24935767Sgibbs/* 25035767Sgibbs * Allocate a piece of memory that can be efficiently mapped into 25135767Sgibbs * bus device space based on the constraints lited in the dma tag. 25235767Sgibbs * A dmamap to for use with dmamap_load is also allocated. 25335767Sgibbs */ 25435767Sgibbsint 255115316Sscottlbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 256259511Skib bus_dmamap_t *mapp) 25735767Sgibbs{ 258259511Skib struct bus_dma_tag_common *tc; 259118081Smux 260259511Skib tc = (struct bus_dma_tag_common *)dmat; 261259511Skib return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp)); 26235767Sgibbs} 26335767Sgibbs 26435767Sgibbs/* 26535767Sgibbs * Free a piece of memory and it's allociated dmamap, that was allocated 26695076Salfred * via bus_dmamem_alloc. Make the same choice for free/contigfree. 26735767Sgibbs */ 26835767Sgibbsvoid 269115316Sscottlbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 27035767Sgibbs{ 271259511Skib struct bus_dma_tag_common *tc; 27235767Sgibbs 273259511Skib tc = (struct bus_dma_tag_common *)dmat; 274259511Skib tc->impl->mem_free(dmat, vaddr, map); 275246713Skib} 276246713Skib 277162211Sscottl/* 278259511Skib * Utility function to load a physical buffer. segp contains 279259511Skib * the starting segment on entrace, and the ending segment on exit. 280246713Skib */ 281259511Skibint 282259511Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 283259511Skib bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 284246713Skib{ 285259511Skib struct bus_dma_tag_common *tc; 286246713Skib 287259511Skib tc = (struct bus_dma_tag_common *)dmat; 288259511Skib return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs, 289259511Skib segp)); 290246713Skib} 291246713Skib 292246713Skibint 293259511Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, 294259511Skib bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, 295259511Skib int *segp) 296246713Skib{ 297259511Skib struct bus_dma_tag_common *tc; 298246713Skib 299259511Skib tc = (struct bus_dma_tag_common *)dmat; 300259511Skib return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags, 301259511Skib segs, segp)); 302246713Skib} 303246713Skib 304246713Skib/* 305246713Skib * Utility function to load a linear buffer. segp contains 306246713Skib * the starting segment on entrace, and the ending segment on exit. 307246713Skib */ 308246713Skibint 309259511Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 310259511Skib bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 311259511Skib int *segp) 312162211Sscottl{ 313259511Skib struct bus_dma_tag_common *tc; 314162211Sscottl 315259511Skib tc = (struct bus_dma_tag_common *)dmat; 316259511Skib return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs, 317259511Skib segp)); 318104486Ssam} 319104486Ssam 320246713Skibvoid 321246713Skib__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 322259511Skib struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 323113459Ssimokawa{ 324259511Skib struct bus_dma_tag_common *tc; 325259511Skib 326259511Skib tc = (struct bus_dma_tag_common *)dmat; 327259511Skib tc->impl->map_waitok(dmat, map, mem, callback, callback_arg); 328113459Ssimokawa} 329113459Ssimokawa 330246713Skibbus_dma_segment_t * 331246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 332259511Skib bus_dma_segment_t *segs, int nsegs, int error) 333104486Ssam{ 334259511Skib struct bus_dma_tag_common *tc; 335104486Ssam 336259511Skib tc = (struct bus_dma_tag_common *)dmat; 337259511Skib return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); 338162275Sscottl} 339162275Sscottl 340104486Ssam/* 34132516Sgibbs * Release the mapping held by map. 34232516Sgibbs */ 34332516Sgibbsvoid 34432516Sgibbs_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 34532516Sgibbs{ 346259511Skib struct bus_dma_tag_common *tc; 34732516Sgibbs 348259511Skib tc = (struct bus_dma_tag_common *)dmat; 349259511Skib tc->impl->map_unload(dmat, map); 35032516Sgibbs} 35132516Sgibbs 35232516Sgibbsvoid 353115343Sscottl_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 35432516Sgibbs{ 355259511Skib struct bus_dma_tag_common *tc; 35632516Sgibbs 357259511Skib tc = (struct bus_dma_tag_common *)dmat; 358259511Skib tc->impl->map_sync(dmat, map, op); 35932516Sgibbs} 360