1282655Szbb/*- 2282655Szbb * Copyright (c) 1997, 1998 Justin T. Gibbs. 3282655Szbb * Copyright (c) 2013, 2015 The FreeBSD Foundation 4282655Szbb * All rights reserved. 5282655Szbb * 6282655Szbb * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7282655Szbb * under sponsorship from the FreeBSD Foundation. 8282655Szbb * 9282655Szbb * Portions of this software were developed by Semihalf 10282655Szbb * under sponsorship of the FreeBSD Foundation. 11282655Szbb * 12282655Szbb * Redistribution and use in source and binary forms, with or without 13282655Szbb * modification, are permitted provided that the following conditions 14282655Szbb * are met: 15282655Szbb * 1. Redistributions of source code must retain the above copyright 16282655Szbb * notice, this list of conditions, and the following disclaimer, 17282655Szbb * without modification, immediately at the beginning of the file. 18282655Szbb * 2. The name of the author may not be used to endorse or promote products 19282655Szbb * derived from this software without specific prior written permission. 20282655Szbb * 21282655Szbb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22282655Szbb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23282655Szbb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24282655Szbb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25282655Szbb * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26282655Szbb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27282655Szbb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28282655Szbb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29282655Szbb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30282655Szbb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31282655Szbb * SUCH DAMAGE. 32282655Szbb */ 33281494Sandrew 34281494Sandrew#include <sys/cdefs.h> 35281494Sandrew__FBSDID("$FreeBSD$"); 36281494Sandrew 37281494Sandrew#include <sys/param.h> 38281494Sandrew#include <sys/systm.h> 39282655Szbb#include <sys/malloc.h> 40282655Szbb#include <sys/bus.h> 41282655Szbb#include <sys/kernel.h> 42282655Szbb#include <sys/ktr.h> 43282655Szbb#include <sys/lock.h> 44282655Szbb#include <sys/memdesc.h> 45282655Szbb#include <sys/mutex.h> 46282655Szbb#include <sys/uio.h> 47281494Sandrew#include <vm/vm.h> 48282655Szbb#include <vm/vm_extern.h> 49281494Sandrew#include <vm/pmap.h> 50281494Sandrew 51281494Sandrew#include <machine/bus.h> 52282655Szbb#include <arm64/include/bus_dma_impl.h> 53281494Sandrew 54282655Szbb/* 55282655Szbb * Convenience function for manipulating driver locks from busdma (during 56282655Szbb * busdma_swi, for example). Drivers that don't provide their own locks 57282655Szbb * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 58282655Szbb * non-mutex locking scheme don't have to use this at all. 59282655Szbb */ 60282655Szbbvoid 61282655Szbbbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 62282655Szbb{ 63282655Szbb struct mtx *dmtx; 64282655Szbb 65282655Szbb dmtx = (struct mtx *)arg; 66282655Szbb switch (op) { 67282655Szbb case BUS_DMA_LOCK: 68282655Szbb mtx_lock(dmtx); 69282655Szbb break; 70282655Szbb case BUS_DMA_UNLOCK: 71282655Szbb mtx_unlock(dmtx); 72282655Szbb break; 73282655Szbb default: 74282655Szbb panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 75282655Szbb } 76282655Szbb} 77282655Szbb 78282655Szbb/* 79282655Szbb * dflt_lock should never get called. It gets put into the dma tag when 80282655Szbb * lockfunc == NULL, which is only valid if the maps that are associated 81282655Szbb * with the tag are meant to never be defered. 82282655Szbb * XXX Should have a way to identify which driver is responsible here. 83282655Szbb */ 84282655Szbbvoid 85282655Szbbbus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op) 86282655Szbb{ 87282655Szbb 88282655Szbb panic("driver error: busdma dflt_lock called"); 89282655Szbb} 90282655Szbb 91282655Szbb/* 92282655Szbb * Return true if a match is made. 93282655Szbb * 94282655Szbb * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 95282655Szbb * 96282655Szbb * If paddr is within the bounds of the dma tag then call the filter callback 97282655Szbb * to check for a match, if there is no filter callback then assume a match. 98282655Szbb */ 99281494Sandrewint 100282655Szbbbus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) 101282655Szbb{ 102282655Szbb int retval; 103282655Szbb 104282655Szbb retval = 0; 105282655Szbb do { 106282655Szbb if (((paddr > tc->lowaddr && paddr <= tc->highaddr) || 107282655Szbb ((paddr & (tc->alignment - 1)) != 0)) && 108282655Szbb (tc->filter == NULL || 109282655Szbb (*tc->filter)(tc->filterarg, paddr) != 0)) 110282655Szbb retval = 1; 111282655Szbb 112282655Szbb tc = tc->parent; 113282655Szbb } while (retval == 0 && tc != NULL); 114282655Szbb return (retval); 115282655Szbb} 116282655Szbb 117282655Szbbint 118282655Szbbcommon_bus_dma_tag_create(struct bus_dma_tag_common *parent, 119282655Szbb bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, 120282655Szbb bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, 121282655Szbb bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 122282655Szbb bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) 123282655Szbb{ 124282655Szbb void *newtag; 125282655Szbb struct bus_dma_tag_common *common; 126282655Szbb 127282655Szbb KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); 128282655Szbb /* Return a NULL tag on failure */ 129282655Szbb *dmat = NULL; 130282655Szbb /* Basic sanity checking */ 131282655Szbb if (boundary != 0 && boundary < maxsegsz) 132282655Szbb maxsegsz = boundary; 133282655Szbb if (maxsegsz == 0) 134282655Szbb return (EINVAL); 135282655Szbb 136282655Szbb newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); 137282655Szbb if (newtag == NULL) { 138282655Szbb CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 139282655Szbb __func__, newtag, 0, ENOMEM); 140282655Szbb return (ENOMEM); 141282655Szbb } 142282655Szbb 143282655Szbb common = newtag; 144282655Szbb common->impl = &bus_dma_bounce_impl; 145282655Szbb common->parent = parent; 146282655Szbb common->alignment = alignment; 147282655Szbb common->boundary = boundary; 148282655Szbb common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 149282655Szbb common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 150282655Szbb common->filter = filter; 151282655Szbb common->filterarg = filterarg; 152282655Szbb common->maxsize = maxsize; 153282655Szbb common->nsegments = nsegments; 154282655Szbb common->maxsegsz = maxsegsz; 155282655Szbb common->flags = flags; 156282655Szbb common->ref_count = 1; /* Count ourself */ 157282655Szbb if (lockfunc != NULL) { 158282655Szbb common->lockfunc = lockfunc; 159282655Szbb common->lockfuncarg = lockfuncarg; 160282655Szbb } else { 161282655Szbb common->lockfunc = bus_dma_dflt_lock; 162282655Szbb common->lockfuncarg = NULL; 163282655Szbb } 164282655Szbb 165282655Szbb /* Take into account any restrictions imposed by our parent tag */ 166282655Szbb if (parent != NULL) { 167282655Szbb common->impl = parent->impl; 168282655Szbb common->lowaddr = MIN(parent->lowaddr, common->lowaddr); 169282655Szbb common->highaddr = MAX(parent->highaddr, common->highaddr); 170282655Szbb if (common->boundary == 0) 171282655Szbb common->boundary = parent->boundary; 172282655Szbb else if (parent->boundary != 0) { 173282655Szbb common->boundary = MIN(parent->boundary, 174282655Szbb common->boundary); 175282655Szbb } 176282655Szbb if (common->filter == NULL) { 177282655Szbb /* 178282655Szbb * Short circuit looking at our parent directly 179282655Szbb * since we have encapsulated all of its information 180282655Szbb */ 181282655Szbb common->filter = parent->filter; 182282655Szbb common->filterarg = parent->filterarg; 183282655Szbb common->parent = parent->parent; 184282655Szbb } 185282655Szbb atomic_add_int(&parent->ref_count, 1); 186282655Szbb } 187282655Szbb *dmat = common; 188282655Szbb return (0); 189282655Szbb} 190282655Szbb 191282655Szbb/* 192282655Szbb * Allocate a device specific dma_tag. 193282655Szbb */ 194282655Szbbint 195282655Szbbbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 196282655Szbb bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 197282655Szbb bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 198282655Szbb int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 199282655Szbb void *lockfuncarg, bus_dma_tag_t *dmat) 200282655Szbb{ 201282655Szbb struct bus_dma_tag_common *tc; 202282655Szbb int error; 203282655Szbb 204282655Szbb if (parent == NULL) { 205282655Szbb error = bus_dma_bounce_impl.tag_create(parent, alignment, 206282655Szbb boundary, lowaddr, highaddr, filter, filterarg, maxsize, 207282655Szbb nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 208282655Szbb } else { 209282655Szbb tc = (struct bus_dma_tag_common *)parent; 210282655Szbb error = tc->impl->tag_create(parent, alignment, 211282655Szbb boundary, lowaddr, highaddr, filter, filterarg, maxsize, 212282655Szbb nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); 213282655Szbb } 214282655Szbb return (error); 215282655Szbb} 216282655Szbb 217282655Szbbint 218282655Szbbbus_dma_tag_destroy(bus_dma_tag_t dmat) 219282655Szbb{ 220282655Szbb struct bus_dma_tag_common *tc; 221282655Szbb 222282655Szbb tc = (struct bus_dma_tag_common *)dmat; 223282655Szbb return (tc->impl->tag_destroy(dmat)); 224282655Szbb} 225282655Szbb 226282655Szbb/* 227282655Szbb * Allocate a handle for mapping from kva/uva/physical 228282655Szbb * address space into bus device space. 229282655Szbb */ 230282655Szbbint 231282655Szbbbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 232282655Szbb{ 233282655Szbb struct bus_dma_tag_common *tc; 234282655Szbb 235282655Szbb tc = (struct bus_dma_tag_common *)dmat; 236282655Szbb return (tc->impl->map_create(dmat, flags, mapp)); 237282655Szbb} 238282655Szbb 239282655Szbb/* 240282655Szbb * Destroy a handle for mapping from kva/uva/physical 241282655Szbb * address space into bus device space. 242282655Szbb */ 243282655Szbbint 244282655Szbbbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 245282655Szbb{ 246282655Szbb struct bus_dma_tag_common *tc; 247282655Szbb 248282655Szbb tc = (struct bus_dma_tag_common *)dmat; 249282655Szbb return (tc->impl->map_destroy(dmat, map)); 250282655Szbb} 251282655Szbb 252282655Szbb 253282655Szbb/* 254282655Szbb * Allocate a piece of memory that can be efficiently mapped into 255282655Szbb * bus device space based on the constraints listed in the dma tag. 256282655Szbb * A dmamap to for use with dmamap_load is also allocated. 257282655Szbb */ 258282655Szbbint 259282655Szbbbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 260282655Szbb bus_dmamap_t *mapp) 261282655Szbb{ 262282655Szbb struct bus_dma_tag_common *tc; 263282655Szbb 264282655Szbb tc = (struct bus_dma_tag_common *)dmat; 265282655Szbb return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp)); 266282655Szbb} 267282655Szbb 268282655Szbb/* 269282655Szbb * Free a piece of memory and it's allociated dmamap, that was allocated 270282655Szbb * via bus_dmamem_alloc. Make the same choice for free/contigfree. 271282655Szbb */ 272282655Szbbvoid 273282655Szbbbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 274282655Szbb{ 275282655Szbb struct bus_dma_tag_common *tc; 276282655Szbb 277282655Szbb tc = (struct bus_dma_tag_common *)dmat; 278282655Szbb tc->impl->mem_free(dmat, vaddr, map); 279282655Szbb} 280282655Szbb 281282655Szbbint 282281494Sandrew_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 283281494Sandrew bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 284281494Sandrew{ 285282655Szbb struct bus_dma_tag_common *tc; 286281494Sandrew 287282655Szbb tc = (struct bus_dma_tag_common *)dmat; 288282655Szbb return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs, 289282655Szbb segp)); 290281494Sandrew} 291281494Sandrew 292281494Sandrewint 293281494Sandrew_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, 294281494Sandrew bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, 295281494Sandrew int *segp) 296281494Sandrew{ 297282655Szbb struct bus_dma_tag_common *tc; 298281494Sandrew 299282655Szbb tc = (struct bus_dma_tag_common *)dmat; 300282655Szbb return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags, 301282655Szbb segs, segp)); 302281494Sandrew} 303281494Sandrew 304281494Sandrewint 305281494Sandrew_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 306281494Sandrew bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 307281494Sandrew int *segp) 308281494Sandrew{ 309282655Szbb struct bus_dma_tag_common *tc; 310281494Sandrew 311282655Szbb tc = (struct bus_dma_tag_common *)dmat; 312282655Szbb return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs, 313282655Szbb segp)); 314281494Sandrew} 315281494Sandrew 316281494Sandrewvoid 317281494Sandrew__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 318281494Sandrew struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 319281494Sandrew{ 320282655Szbb struct bus_dma_tag_common *tc; 321281494Sandrew 322282655Szbb tc = (struct bus_dma_tag_common *)dmat; 323282655Szbb tc->impl->map_waitok(dmat, map, mem, callback, callback_arg); 324281494Sandrew} 325281494Sandrew 326281494Sandrewbus_dma_segment_t * 327281494Sandrew_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 328281494Sandrew bus_dma_segment_t *segs, int nsegs, int error) 329281494Sandrew{ 330282655Szbb struct bus_dma_tag_common *tc; 331281494Sandrew 332282655Szbb tc = (struct bus_dma_tag_common *)dmat; 333282655Szbb return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); 334281494Sandrew} 335281494Sandrew 336282655Szbb/* 337282655Szbb * Release the mapping held by map. 338282655Szbb */ 339281494Sandrewvoid 340281494Sandrew_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 341281494Sandrew{ 342282655Szbb struct bus_dma_tag_common *tc; 343281494Sandrew 344282655Szbb tc = (struct bus_dma_tag_common *)dmat; 345282655Szbb tc->impl->map_unload(dmat, map); 346281494Sandrew} 347281494Sandrew 348281494Sandrewvoid 349281494Sandrew_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 350281494Sandrew{ 351282655Szbb struct bus_dma_tag_common *tc; 352281494Sandrew 353282655Szbb tc = (struct bus_dma_tag_common *)dmat; 354282655Szbb tc->impl->map_sync(dmat, map, op); 355281494Sandrew} 356