busdma_machdep-v4.c revision 289851
1139735Simp/*- 2244471Scognet * Copyright (c) 2012 Ian Lepore 3129198Scognet * Copyright (c) 2004 Olivier Houchard 4129198Scognet * Copyright (c) 2002 Peter Grehan 5129198Scognet * Copyright (c) 1997, 1998 Justin T. Gibbs. 6129198Scognet * All rights reserved. 7129198Scognet * 8129198Scognet * Redistribution and use in source and binary forms, with or without 9129198Scognet * modification, are permitted provided that the following conditions 10129198Scognet * are met: 11129198Scognet * 1. Redistributions of source code must retain the above copyright 12129198Scognet * notice, this list of conditions, and the following disclaimer, 13129198Scognet * without modification, immediately at the beginning of the file. 14129198Scognet * 2. The name of the author may not be used to endorse or promote products 15129198Scognet * derived from this software without specific prior written permission. 16129198Scognet * 17129198Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18129198Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19129198Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20129198Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21129198Scognet * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22129198Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23129198Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24129198Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25129198Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26129198Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27129198Scognet * SUCH DAMAGE. 28129198Scognet * 29129198Scognet * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30129198Scognet */ 31129198Scognet 32129198Scognet#include <sys/cdefs.h> 33129198Scognet__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289851 2015-10-23 20:49:34Z ian $"); 34129198Scognet 35129198Scognet/* 36244471Scognet * ARM bus dma support routines. 37244471Scognet * 38244471Scognet * XXX Things to investigate / fix some day... 39244471Scognet * - What is the earliest that this API can be called? Could there be any 40244471Scognet * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41244471Scognet * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42244471Scognet * bus_dmamap_load() function. This code has historically (and still does) 43244471Scognet * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44244471Scognet * error checking because some resource management calls would become WAITOK 45244471Scognet * and thus "cannot fail." 46244471Scognet * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47244471Scognet * creation time, and the result stored in the tag. 48244471Scognet * - It should be possible to take some shortcuts when mapping a buffer we know 49244471Scognet * came from the uma(9) allocators based on what we know about such buffers 50244471Scognet * (aligned, contiguous, etc). 51244471Scognet * - The allocation of bounce pages could probably be cleaned up, then we could 52244471Scognet * retire arm_remap_nocache(). 53129198Scognet */ 54129198Scognet 55129198Scognet#define _ARM32_BUS_DMA_PRIVATE 56129198Scognet#include <sys/param.h> 57129198Scognet#include <sys/systm.h> 58129198Scognet#include <sys/malloc.h> 59129198Scognet#include <sys/bus.h> 60244471Scognet#include <sys/busdma_bufalloc.h> 61129198Scognet#include <sys/interrupt.h> 62129198Scognet#include <sys/lock.h> 63129198Scognet#include <sys/proc.h> 64246713Skib#include <sys/memdesc.h> 65129198Scognet#include <sys/mutex.h> 66140310Scognet#include <sys/ktr.h> 67146597Scognet#include <sys/kernel.h> 68166063Scognet#include <sys/sysctl.h> 69246713Skib#include <sys/uio.h> 70129198Scognet 71244471Scognet#include <vm/uma.h> 72129198Scognet#include <vm/vm.h> 73244471Scognet#include <vm/vm_extern.h> 74244471Scognet#include <vm/vm_kern.h> 75129198Scognet#include <vm/vm_page.h> 76129198Scognet#include <vm/vm_map.h> 77129198Scognet 78129198Scognet#include <machine/atomic.h> 79129198Scognet#include <machine/bus.h> 80129198Scognet#include <machine/cpufunc.h> 81166063Scognet#include <machine/md_var.h> 82129198Scognet 83289851Sian#define MAX_BPAGES 64 84289851Sian#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85289851Sian#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86166063Scognet 87166063Scognetstruct bounce_zone; 88166063Scognet 89129198Scognetstruct bus_dma_tag { 90129198Scognet bus_dma_tag_t parent; 91129198Scognet bus_size_t alignment; 92232356Sjhb bus_addr_t boundary; 93129198Scognet bus_addr_t lowaddr; 94129198Scognet bus_addr_t highaddr; 95129198Scognet bus_dma_filter_t *filter; 96129198Scognet void *filterarg; 97129198Scognet bus_size_t maxsize; 98129198Scognet u_int nsegments; 99129198Scognet bus_size_t maxsegsz; 100129198Scognet int flags; 101129198Scognet int ref_count; 102129198Scognet int map_count; 103129198Scognet bus_dma_lock_t *lockfunc; 104129198Scognet void *lockfuncarg; 105289851Sian struct bounce_zone *bounce_zone; 106129198Scognet /* 107129198Scognet * DMA range for this tag. If the page doesn't fall within 108129198Scognet * one of these ranges, an error is returned. The caller 109129198Scognet * may then decide what to do with the transfer. If the 110129198Scognet * range pointer is NULL, it is ignored. 111129198Scognet */ 112129198Scognet struct arm32_dma_range *ranges; 113129198Scognet int _nranges; 114244471Scognet /* 115244471Scognet * Most tags need one or two segments, and can use the local tagsegs 116244471Scognet * array. For tags with a larger limit, we'll allocate a bigger array 117244471Scognet * on first use. 118244471Scognet */ 119244471Scognet bus_dma_segment_t *segments; 120244471Scognet bus_dma_segment_t tagsegs[2]; 121129198Scognet}; 122129198Scognet 123166063Scognetstruct bounce_page { 124166063Scognet vm_offset_t vaddr; /* kva of bounce buffer */ 125166063Scognet bus_addr_t busaddr; /* Physical address */ 126166063Scognet vm_offset_t datavaddr; /* kva of client data */ 127289675Sjah vm_page_t datapage; /* physical page of client data */ 128289675Sjah vm_offset_t dataoffs; /* page offset of client data */ 129166063Scognet bus_size_t datacount; /* client data count */ 130166063Scognet STAILQ_ENTRY(bounce_page) links; 131166063Scognet}; 132166063Scognet 133246713Skibstruct sync_list { 134289675Sjah vm_offset_t vaddr; /* kva of client data */ 135289675Sjah vm_page_t pages; /* starting page of client data */ 136289675Sjah vm_offset_t dataoffs; /* page offset of client data */ 137246713Skib bus_size_t datacount; /* client data count */ 138246713Skib}; 139246713Skib 140166063Scognetint busdma_swi_pending; 141166063Scognet 142166063Scognetstruct bounce_zone { 143166063Scognet STAILQ_ENTRY(bounce_zone) links; 144166063Scognet STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 145166063Scognet int total_bpages; 146166063Scognet int free_bpages; 147166063Scognet int reserved_bpages; 148166063Scognet int active_bpages; 149166063Scognet int total_bounced; 150166063Scognet int total_deferred; 151188403Scognet int map_count; 152166063Scognet bus_size_t alignment; 153166063Scognet bus_addr_t lowaddr; 154166063Scognet char zoneid[8]; 155166063Scognet char lowaddrid[20]; 156166063Scognet struct sysctl_ctx_list sysctl_tree; 157166063Scognet struct sysctl_oid *sysctl_tree_top; 158166063Scognet}; 159166063Scognet 160166063Scognetstatic struct mtx bounce_lock; 161166063Scognetstatic int total_bpages; 162166063Scognetstatic int busdma_zonecount; 163166063Scognetstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 164166063Scognet 165227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 166166063ScognetSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 167166063Scognet "Total bounce pages"); 168166063Scognet 169129198Scognetstruct bus_dmamap { 170289851Sian struct bp_list bpages; 171289851Sian int pagesneeded; 172289851Sian int pagesreserved; 173289851Sian bus_dma_tag_t dmat; 174289851Sian struct memdesc mem; 175289851Sian bus_dmamap_callback_t *callback; 176289851Sian void *callback_arg; 177289851Sian int flags; 178289851Sian#define DMAMAP_COHERENT 0x8 179289851Sian#define DMAMAP_CACHE_ALIGNED 0x10 180166063Scognet STAILQ_ENTRY(bus_dmamap) links; 181289851Sian int sync_count; 182289851Sian struct sync_list *slist; 183129198Scognet}; 184129198Scognet 185166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 186166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 187166063Scognet 188146597Scognetstatic struct mtx busdma_mtx; 189146597Scognet 190146597ScognetMTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 191146597Scognet 192166063Scognetstatic void init_bounce_pages(void *dummy); 193166063Scognetstatic int alloc_bounce_zone(bus_dma_tag_t dmat); 194166063Scognetstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 195166063Scognetstatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 196289851Sian int commit); 197166063Scognetstatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 198289851Sian vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 199166063Scognetstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200289675Sjahstatic void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 201289851Sian int bufaligned); 202166063Scognet 203166063Scognet/* Default tag, as most drivers provide no parent tag. */ 204166063Scognetbus_dma_tag_t arm_root_dma_tag; 205166063Scognet 206244473Scognet/* 207244473Scognet * ---------------------------------------------------------------------------- 208244473Scognet * Begin block of code useful to transplant to other implementations. 209244473Scognet */ 210244471Scognet 211244471Scognetstatic uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 212244471Scognet 213244471Scognetstatic busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 214244471Scognetstatic busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 215244471Scognet 216166063Scognet/* 217244471Scognet * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 218244471Scognet * It'll need platform-specific changes if this code is copied. 219244471Scognet */ 220244471Scognetstatic int 221244471Scognetdmamap_ctor(void *mem, int size, void *arg, int flags) 222244471Scognet{ 223244471Scognet bus_dmamap_t map; 224244471Scognet bus_dma_tag_t dmat; 225244471Scognet 226244471Scognet map = (bus_dmamap_t)mem; 227244471Scognet dmat = (bus_dma_tag_t)arg; 228244471Scognet 229244471Scognet dmat->map_count++; 230244471Scognet 231244471Scognet map->dmat = dmat; 232244471Scognet map->flags = 0; 233244471Scognet STAILQ_INIT(&map->bpages); 234244471Scognet 235244471Scognet return (0); 236244471Scognet} 237244471Scognet 238244471Scognet/* 239244471Scognet * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 240244471Scognet * It may need platform-specific changes if this code is copied . 241244471Scognet */ 242283366Sandrewstatic void 243244471Scognetdmamap_dtor(void *mem, int size, void *arg) 244244471Scognet{ 245244471Scognet bus_dmamap_t map; 246244471Scognet 247244471Scognet map = (bus_dmamap_t)mem; 248244471Scognet 249244471Scognet map->dmat->map_count--; 250244471Scognet} 251244471Scognet 252244471Scognetstatic void 253244471Scognetbusdma_init(void *dummy) 254244471Scognet{ 255244471Scognet 256244471Scognet /* Create a cache of maps for bus_dmamap_create(). */ 257244471Scognet dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 258244471Scognet dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 259244471Scognet 260244471Scognet /* Create a cache of buffers in standard (cacheable) memory. */ 261283366Sandrew standard_allocator = busdma_bufalloc_create("buffer", 262244471Scognet arm_dcache_align, /* minimum_alignment */ 263283366Sandrew NULL, /* uma_alloc func */ 264244471Scognet NULL, /* uma_free func */ 265244471Scognet 0); /* uma_zcreate_flags */ 266244471Scognet 267244471Scognet /* 268244471Scognet * Create a cache of buffers in uncacheable memory, to implement the 269244471Scognet * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 270244471Scognet */ 271244471Scognet coherent_allocator = busdma_bufalloc_create("coherent", 272244471Scognet arm_dcache_align, /* minimum_alignment */ 273283366Sandrew busdma_bufalloc_alloc_uncacheable, 274283366Sandrew busdma_bufalloc_free_uncacheable, 275244471Scognet 0); /* uma_zcreate_flags */ 276244471Scognet} 277244471Scognet 278244471Scognet/* 279244471Scognet * This init historically used SI_SUB_VM, but now the init code requires 280244471Scognet * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 281267992Shselasky * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 282267992Shselasky * SI_SUB_KMEM and SI_ORDER_FOURTH. 283244471Scognet */ 284267992ShselaskySYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 285244471Scognet 286244473Scognet/* 287244473Scognet * End block of code useful to transplant to other implementations. 288244473Scognet * ---------------------------------------------------------------------------- 289244473Scognet */ 290244471Scognet 291244471Scognet/* 292166063Scognet * Return true if a match is made. 293166063Scognet * 294166063Scognet * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 295166063Scognet * 296166063Scognet * If paddr is within the bounds of the dma tag then call the filter callback 297166063Scognet * to check for a match, if there is no filter callback then assume a match. 298166063Scognet */ 299166063Scognetstatic int 300166063Scognetrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 301166063Scognet{ 302166063Scognet int retval; 303166063Scognet 304166063Scognet retval = 0; 305166063Scognet 306166063Scognet do { 307166063Scognet if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 308166063Scognet || ((paddr & (dmat->alignment - 1)) != 0)) 309166063Scognet && (dmat->filter == NULL 310166063Scognet || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 311166063Scognet retval = 1; 312166063Scognet 313283366Sandrew dmat = dmat->parent; 314166063Scognet } while (retval == 0 && dmat != NULL); 315166063Scognet return (retval); 316166063Scognet} 317166063Scognet 318129198Scognet/* 319244471Scognet * This routine checks the exclusion zone constraints from a tag against the 320244471Scognet * physical RAM available on the machine. If a tag specifies an exclusion zone 321244471Scognet * but there's no RAM in that zone, then we avoid allocating resources to bounce 322244471Scognet * a request, and we can use any memory allocator (as opposed to needing 323244471Scognet * kmem_alloc_contig() just because it can allocate pages in an address range). 324244471Scognet * 325244471Scognet * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 326244471Scognet * same value on 32-bit architectures) as their lowaddr constraint, and we can't 327244471Scognet * possibly have RAM at an address higher than the highest address we can 328244471Scognet * express, so we take a fast out. 329129198Scognet */ 330137758Scognetstatic __inline int 331166063Scognet_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 332166063Scognet{ 333166063Scognet int i; 334244471Scognet 335244471Scognet if (lowaddr >= BUS_SPACE_MAXADDR) 336244471Scognet return (0); 337244471Scognet 338166063Scognet for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 339166063Scognet if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 340236991Simp || (lowaddr < phys_avail[i] && 341166063Scognet highaddr > phys_avail[i])) 342166063Scognet return (1); 343166063Scognet } 344166063Scognet return (0); 345166063Scognet} 346166063Scognet 347129198Scognetstatic __inline struct arm32_dma_range * 348129198Scognet_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 349129198Scognet bus_addr_t curaddr) 350129198Scognet{ 351129198Scognet struct arm32_dma_range *dr; 352129198Scognet int i; 353129198Scognet 354129198Scognet for (i = 0, dr = ranges; i < nranges; i++, dr++) { 355129198Scognet if (curaddr >= dr->dr_sysbase && 356129198Scognet round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 357129198Scognet return (dr); 358129198Scognet } 359129198Scognet 360129198Scognet return (NULL); 361129198Scognet} 362289851Sian 363129198Scognet/* 364129198Scognet * Convenience function for manipulating driver locks from busdma (during 365129198Scognet * busdma_swi, for example). Drivers that don't provide their own locks 366129198Scognet * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 367129198Scognet * non-mutex locking scheme don't have to use this at all. 368129198Scognet */ 369129198Scognetvoid 370129198Scognetbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 371129198Scognet{ 372129198Scognet struct mtx *dmtx; 373129198Scognet 374129198Scognet dmtx = (struct mtx *)arg; 375129198Scognet switch (op) { 376129198Scognet case BUS_DMA_LOCK: 377129198Scognet mtx_lock(dmtx); 378129198Scognet break; 379129198Scognet case BUS_DMA_UNLOCK: 380129198Scognet mtx_unlock(dmtx); 381129198Scognet break; 382129198Scognet default: 383129198Scognet panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 384129198Scognet } 385129198Scognet} 386129198Scognet 387129198Scognet/* 388129198Scognet * dflt_lock should never get called. It gets put into the dma tag when 389129198Scognet * lockfunc == NULL, which is only valid if the maps that are associated 390129198Scognet * with the tag are meant to never be defered. 391129198Scognet * XXX Should have a way to identify which driver is responsible here. 392129198Scognet */ 393129198Scognetstatic void 394129198Scognetdflt_lock(void *arg, bus_dma_lock_op_t op) 395129198Scognet{ 396129198Scognet#ifdef INVARIANTS 397129198Scognet panic("driver error: busdma dflt_lock called"); 398129198Scognet#else 399129198Scognet printf("DRIVER_ERROR: busdma dflt_lock called\n"); 400129198Scognet#endif 401129198Scognet} 402129198Scognet 403129198Scognet/* 404129198Scognet * Allocate a device specific dma_tag. 405129198Scognet */ 406135644Scognet#define SEG_NB 1024 407135644Scognet 408129198Scognetint 409129198Scognetbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 410289851Sian bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 411289851Sian bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 412289851Sian int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 413289851Sian void *lockfuncarg, bus_dma_tag_t *dmat) 414129198Scognet{ 415129198Scognet bus_dma_tag_t newtag; 416129198Scognet int error = 0; 417129198Scognet /* Return a NULL tag on failure */ 418129198Scognet *dmat = NULL; 419166063Scognet if (!parent) 420166063Scognet parent = arm_root_dma_tag; 421129198Scognet 422129198Scognet newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 423140313Scognet if (newtag == NULL) { 424143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 425143284Smux __func__, newtag, 0, error); 426129198Scognet return (ENOMEM); 427140313Scognet } 428129198Scognet 429129198Scognet newtag->parent = parent; 430244471Scognet newtag->alignment = alignment ? alignment : 1; 431129198Scognet newtag->boundary = boundary; 432129198Scognet newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 433129198Scognet newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 434129198Scognet newtag->filter = filter; 435129198Scognet newtag->filterarg = filterarg; 436289851Sian newtag->maxsize = maxsize; 437289851Sian newtag->nsegments = nsegments; 438129198Scognet newtag->maxsegsz = maxsegsz; 439129198Scognet newtag->flags = flags; 440129198Scognet newtag->ref_count = 1; /* Count ourself */ 441129198Scognet newtag->map_count = 0; 442129198Scognet newtag->ranges = bus_dma_get_range(); 443135644Scognet newtag->_nranges = bus_dma_get_range_nb(); 444129198Scognet if (lockfunc != NULL) { 445129198Scognet newtag->lockfunc = lockfunc; 446129198Scognet newtag->lockfuncarg = lockfuncarg; 447129198Scognet } else { 448129198Scognet newtag->lockfunc = dflt_lock; 449129198Scognet newtag->lockfuncarg = NULL; 450129198Scognet } 451244471Scognet /* 452244471Scognet * If all the segments we need fit into the local tagsegs array, set the 453244471Scognet * pointer now. Otherwise NULL the pointer and an array of segments 454244471Scognet * will be allocated later, on first use. We don't pre-allocate now 455244471Scognet * because some tags exist just to pass contraints to children in the 456244471Scognet * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 457244471Scognet * sure don't want to try to allocate an array for that. 458244471Scognet */ 459244471Scognet if (newtag->nsegments <= nitems(newtag->tagsegs)) 460244471Scognet newtag->segments = newtag->tagsegs; 461244471Scognet else 462244471Scognet newtag->segments = NULL; 463244471Scognet /* 464129198Scognet * Take into account any restrictions imposed by our parent tag 465129198Scognet */ 466289851Sian if (parent != NULL) { 467289851Sian newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 468289851Sian newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 469134934Sscottl if (newtag->boundary == 0) 470134934Sscottl newtag->boundary = parent->boundary; 471134934Sscottl else if (parent->boundary != 0) 472289851Sian newtag->boundary = MIN(parent->boundary, 473134934Sscottl newtag->boundary); 474166063Scognet if ((newtag->filter != NULL) || 475166063Scognet ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 476166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 477289851Sian if (newtag->filter == NULL) { 478289851Sian /* 479289851Sian * Short circuit looking at our parent directly 480289851Sian * since we have encapsulated all of its information 481289851Sian */ 482289851Sian newtag->filter = parent->filter; 483289851Sian newtag->filterarg = parent->filterarg; 484289851Sian newtag->parent = parent->parent; 485129198Scognet } 486129198Scognet if (newtag->parent != NULL) 487129198Scognet atomic_add_int(&parent->ref_count, 1); 488129198Scognet } 489166063Scognet if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 490166063Scognet || newtag->alignment > 1) 491166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 492129198Scognet 493166063Scognet if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 494166063Scognet (flags & BUS_DMA_ALLOCNOW) != 0) { 495166063Scognet struct bounce_zone *bz; 496166063Scognet 497166063Scognet /* Must bounce */ 498166063Scognet 499166063Scognet if ((error = alloc_bounce_zone(newtag)) != 0) { 500166063Scognet free(newtag, M_DEVBUF); 501166063Scognet return (error); 502166063Scognet } 503166063Scognet bz = newtag->bounce_zone; 504166063Scognet 505166063Scognet if (ptoa(bz->total_bpages) < maxsize) { 506166063Scognet int pages; 507166063Scognet 508166063Scognet pages = atop(maxsize) - bz->total_bpages; 509166063Scognet 510166063Scognet /* Add pages to our bounce pool */ 511166063Scognet if (alloc_bounce_pages(newtag, pages) < pages) 512166063Scognet error = ENOMEM; 513166063Scognet } 514166063Scognet /* Performed initial allocation */ 515166063Scognet newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 516170502Scognet } else 517170502Scognet newtag->bounce_zone = NULL; 518166063Scognet if (error != 0) 519166063Scognet free(newtag, M_DEVBUF); 520166063Scognet else 521166063Scognet *dmat = newtag; 522143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 523143284Smux __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 524140313Scognet 525129198Scognet return (error); 526129198Scognet} 527129198Scognet 528129198Scognetint 529129198Scognetbus_dma_tag_destroy(bus_dma_tag_t dmat) 530129198Scognet{ 531140680Scognet#ifdef KTR 532140313Scognet bus_dma_tag_t dmat_copy = dmat; 533140680Scognet#endif 534140313Scognet 535129198Scognet if (dmat != NULL) { 536283366Sandrew 537289851Sian if (dmat->map_count != 0) 538289851Sian return (EBUSY); 539283366Sandrew 540289851Sian while (dmat != NULL) { 541289851Sian bus_dma_tag_t parent; 542283366Sandrew 543289851Sian parent = dmat->parent; 544289851Sian atomic_subtract_int(&dmat->ref_count, 1); 545289851Sian if (dmat->ref_count == 0) { 546244471Scognet if (dmat->segments != NULL && 547244471Scognet dmat->segments != dmat->tagsegs) 548240177Sjhb free(dmat->segments, M_DEVBUF); 549289851Sian free(dmat, M_DEVBUF); 550289851Sian /* 551289851Sian * Last reference count, so 552289851Sian * release our reference 553289851Sian * count on our parent. 554289851Sian */ 555289851Sian dmat = parent; 556289851Sian } else 557289851Sian dmat = NULL; 558289851Sian } 559289851Sian } 560143294Smux CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 561140313Scognet 562289851Sian return (0); 563129198Scognet} 564129198Scognet 565166063Scognet#include <sys/kdb.h> 566129198Scognet/* 567129198Scognet * Allocate a handle for mapping from kva/uva/physical 568129198Scognet * address space into bus device space. 569129198Scognet */ 570129198Scognetint 571129198Scognetbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 572129198Scognet{ 573246713Skib struct sync_list *slist; 574244471Scognet bus_dmamap_t map; 575140313Scognet int error = 0; 576129198Scognet 577246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 578246713Skib if (slist == NULL) 579246713Skib return (ENOMEM); 580246713Skib 581244575Scognet map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 582244471Scognet *mapp = map; 583246713Skib if (map == NULL) { 584246713Skib free(slist, M_DEVBUF); 585244575Scognet return (ENOMEM); 586246713Skib } 587240177Sjhb 588244471Scognet /* 589244471Scognet * If the tag's segments haven't been allocated yet we need to do it 590244471Scognet * now, because we can't sleep for resources at map load time. 591244471Scognet */ 592244575Scognet if (dmat->segments == NULL) { 593283366Sandrew dmat->segments = malloc(dmat->nsegments * 594244575Scognet sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 595244575Scognet if (dmat->segments == NULL) { 596246713Skib free(slist, M_DEVBUF); 597244575Scognet uma_zfree(dmamap_zone, map); 598244575Scognet *mapp = NULL; 599244575Scognet return (ENOMEM); 600244575Scognet } 601244575Scognet } 602129198Scognet 603166063Scognet /* 604166063Scognet * Bouncing might be required if the driver asks for an active 605166063Scognet * exclusion region, a data alignment that is stricter than 1, and/or 606166063Scognet * an active address boundary. 607166063Scognet */ 608166063Scognet if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 609166063Scognet 610166063Scognet /* Must bounce */ 611166063Scognet struct bounce_zone *bz; 612166063Scognet int maxpages; 613166063Scognet 614166063Scognet if (dmat->bounce_zone == NULL) { 615166063Scognet if ((error = alloc_bounce_zone(dmat)) != 0) { 616246713Skib free(slist, M_DEVBUF); 617244471Scognet uma_zfree(dmamap_zone, map); 618166063Scognet *mapp = NULL; 619166063Scognet return (error); 620166063Scognet } 621166063Scognet } 622166063Scognet bz = dmat->bounce_zone; 623166063Scognet 624166063Scognet /* Initialize the new map */ 625166063Scognet STAILQ_INIT(&((*mapp)->bpages)); 626166063Scognet 627166063Scognet /* 628166063Scognet * Attempt to add pages to our pool on a per-instance 629166063Scognet * basis up to a sane limit. 630166063Scognet */ 631166063Scognet maxpages = MAX_BPAGES; 632166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 633188403Scognet || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 634166063Scognet int pages; 635166063Scognet 636166063Scognet pages = MAX(atop(dmat->maxsize), 1); 637166063Scognet pages = MIN(maxpages - bz->total_bpages, pages); 638166063Scognet pages = MAX(pages, 1); 639166063Scognet if (alloc_bounce_pages(dmat, pages) < pages) 640166063Scognet error = ENOMEM; 641166063Scognet 642166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 643166063Scognet if (error == 0) 644166063Scognet dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 645166063Scognet } else { 646166063Scognet error = 0; 647166063Scognet } 648166063Scognet } 649188403Scognet bz->map_count++; 650166063Scognet } 651246713Skib map->sync_count = 0; 652246713Skib map->slist = slist; 653143294Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 654143284Smux __func__, dmat, dmat->flags, error); 655140313Scognet 656129198Scognet return (0); 657129198Scognet} 658129198Scognet 659129198Scognet/* 660129198Scognet * Destroy a handle for mapping from kva/uva/physical 661129198Scognet * address space into bus device space. 662129198Scognet */ 663129198Scognetint 664129198Scognetbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 665129198Scognet{ 666135644Scognet 667246713Skib if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 668166063Scognet CTR3(KTR_BUSDMA, "%s: tag %p error %d", 669166063Scognet __func__, dmat, EBUSY); 670166063Scognet return (EBUSY); 671166063Scognet } 672246713Skib free(map->slist, M_DEVBUF); 673244471Scognet uma_zfree(dmamap_zone, map); 674188403Scognet if (dmat->bounce_zone) 675188403Scognet dmat->bounce_zone->map_count--; 676143294Smux CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 677129198Scognet return (0); 678129198Scognet} 679129198Scognet 680129198Scognet/* 681244471Scognet * Allocate a piece of memory that can be efficiently mapped into bus device 682244471Scognet * space based on the constraints listed in the dma tag. Returns a pointer to 683244471Scognet * the allocated memory, and a pointer to an associated bus_dmamap. 684129198Scognet */ 685129198Scognetint 686244471Scognetbus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 687289851Sian bus_dmamap_t *mapp) 688129198Scognet{ 689246713Skib struct sync_list *slist; 690244471Scognet void * vaddr; 691244471Scognet struct busdma_bufzone *bufzone; 692244471Scognet busdma_bufalloc_t ba; 693244471Scognet bus_dmamap_t map; 694129198Scognet int mflags; 695244471Scognet vm_memattr_t memattr; 696129198Scognet 697129198Scognet if (flags & BUS_DMA_NOWAIT) 698129198Scognet mflags = M_NOWAIT; 699129198Scognet else 700129198Scognet mflags = M_WAITOK; 701244471Scognet /* 702244471Scognet * If the tag's segments haven't been allocated yet we need to do it 703244471Scognet * now, because we can't sleep for resources at map load time. 704244471Scognet */ 705244471Scognet if (dmat->segments == NULL) 706283366Sandrew dmat->segments = malloc(dmat->nsegments * 707244471Scognet sizeof(*dmat->segments), M_DEVBUF, mflags); 708244471Scognet 709246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 710246713Skib if (slist == NULL) 711246713Skib return (ENOMEM); 712244471Scognet map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 713246713Skib if (map == NULL) { 714246713Skib free(slist, M_DEVBUF); 715244471Scognet return (ENOMEM); 716246713Skib } 717244471Scognet if (flags & BUS_DMA_COHERENT) { 718244471Scognet memattr = VM_MEMATTR_UNCACHEABLE; 719244471Scognet ba = coherent_allocator; 720244471Scognet map->flags |= DMAMAP_COHERENT; 721244471Scognet } else { 722244471Scognet memattr = VM_MEMATTR_DEFAULT; 723244471Scognet ba = standard_allocator; 724240177Sjhb } 725244471Scognet /* All buffers we allocate are cache-aligned. */ 726244471Scognet map->flags |= DMAMAP_CACHE_ALIGNED; 727244471Scognet 728129198Scognet if (flags & BUS_DMA_ZERO) 729129198Scognet mflags |= M_ZERO; 730129198Scognet 731244471Scognet /* 732244471Scognet * Try to find a bufzone in the allocator that holds a cache of buffers 733244471Scognet * of the right size for this request. If the buffer is too big to be 734244471Scognet * held in the allocator cache, this returns NULL. 735244471Scognet */ 736244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 737244471Scognet 738244471Scognet /* 739244471Scognet * Allocate the buffer from the uma(9) allocator if... 740244471Scognet * - It's small enough to be in the allocator (bufzone not NULL). 741244471Scognet * - The alignment constraint isn't larger than the allocation size 742244471Scognet * (the allocator aligns buffers to their size boundaries). 743244471Scognet * - There's no need to handle lowaddr/highaddr exclusion zones. 744244471Scognet * else allocate non-contiguous pages if... 745244471Scognet * - The page count that could get allocated doesn't exceed nsegments. 746244471Scognet * - The alignment constraint isn't larger than a page boundary. 747244471Scognet * - There are no boundary-crossing constraints. 748244471Scognet * else allocate a block of contiguous pages because one or more of the 749244471Scognet * constraints is something that only the contig allocator can fulfill. 750244471Scognet */ 751244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 752244471Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 753244471Scognet vaddr = uma_zalloc(bufzone->umazone, mflags); 754244471Scognet } else if (dmat->nsegments >= btoc(dmat->maxsize) && 755244471Scognet dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 756254025Sjeff vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 757244471Scognet mflags, 0, dmat->lowaddr, memattr); 758244471Scognet } else { 759254025Sjeff vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 760244471Scognet mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 761244471Scognet memattr); 762135644Scognet } 763244471Scognet if (vaddr == NULL) { 764246713Skib free(slist, M_DEVBUF); 765244471Scognet uma_zfree(dmamap_zone, map); 766244471Scognet map = NULL; 767246713Skib } else { 768246713Skib map->slist = slist; 769246713Skib map->sync_count = 0; 770129198Scognet } 771244471Scognet *vaddrp = vaddr; 772244471Scognet *mapp = map; 773244471Scognet 774244471Scognet return (vaddr == NULL ? ENOMEM : 0); 775129198Scognet} 776129198Scognet 777129198Scognet/* 778244471Scognet * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 779244471Scognet * its associated map. 780129198Scognet */ 781129198Scognetvoid 782129198Scognetbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 783129198Scognet{ 784244471Scognet struct busdma_bufzone *bufzone; 785244471Scognet busdma_bufalloc_t ba; 786244471Scognet 787244471Scognet if (map->flags & DMAMAP_COHERENT) 788244471Scognet ba = coherent_allocator; 789246713Skib else 790244471Scognet ba = standard_allocator; 791244471Scognet 792246713Skib free(map->slist, M_DEVBUF); 793289619Sian uma_zfree(dmamap_zone, map); 794244471Scognet 795244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 796244471Scognet 797244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 798166063Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 799244471Scognet uma_zfree(bufzone->umazone, vaddr); 800244471Scognet else 801254025Sjeff kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 802129198Scognet} 803129198Scognet 804246713Skibstatic void 805246713Skib_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 806246713Skib bus_size_t buflen, int flags) 807246713Skib{ 808246713Skib bus_addr_t curaddr; 809246713Skib bus_size_t sgsize; 810246713Skib 811257203Sian if (map->pagesneeded == 0) { 812246713Skib CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 813246713Skib dmat->lowaddr, dmat->boundary, dmat->alignment); 814246713Skib CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 815246713Skib map, map->pagesneeded); 816246713Skib /* 817246713Skib * Count the number of bounce pages 818246713Skib * needed in order to complete this transfer 819246713Skib */ 820246713Skib curaddr = buf; 821246713Skib while (buflen != 0) { 822246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 823246713Skib if (run_filter(dmat, curaddr) != 0) { 824289675Sjah sgsize = MIN(sgsize, 825289675Sjah PAGE_SIZE - (curaddr & PAGE_MASK)); 826246713Skib map->pagesneeded++; 827246713Skib } 828246713Skib curaddr += sgsize; 829246713Skib buflen -= sgsize; 830246713Skib } 831246713Skib CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 832246713Skib } 833246713Skib} 834246713Skib 835246713Skibstatic void 836191011Skib_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 837191011Skib void *buf, bus_size_t buflen, int flags) 838166063Scognet{ 839166063Scognet vm_offset_t vaddr; 840166063Scognet vm_offset_t vendaddr; 841166063Scognet bus_addr_t paddr; 842166063Scognet 843257203Sian if (map->pagesneeded == 0) { 844185494Sstas CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 845185494Sstas dmat->lowaddr, dmat->boundary, dmat->alignment); 846170406Scognet CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 847170406Scognet map, map->pagesneeded); 848166063Scognet /* 849166063Scognet * Count the number of bounce pages 850166063Scognet * needed in order to complete this transfer 851166063Scognet */ 852166063Scognet vaddr = trunc_page((vm_offset_t)buf); 853166063Scognet vendaddr = (vm_offset_t)buf + buflen; 854166063Scognet 855166063Scognet while (vaddr < vendaddr) { 856246713Skib if (__predict_true(pmap == kernel_pmap)) 857191438Sjhb paddr = pmap_kextract(vaddr); 858191438Sjhb else 859191011Skib paddr = pmap_extract(pmap, vaddr); 860246713Skib if (run_filter(dmat, paddr) != 0) 861166063Scognet map->pagesneeded++; 862166063Scognet vaddr += PAGE_SIZE; 863166063Scognet } 864166063Scognet CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 865166063Scognet } 866246713Skib} 867166063Scognet 868246713Skibstatic int 869246713Skib_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 870246713Skib{ 871246713Skib 872166063Scognet /* Reserve Necessary Bounce Pages */ 873246713Skib mtx_lock(&bounce_lock); 874246713Skib if (flags & BUS_DMA_NOWAIT) { 875246713Skib if (reserve_bounce_pages(dmat, map, 0) != 0) { 876246713Skib mtx_unlock(&bounce_lock); 877246713Skib return (ENOMEM); 878166063Scognet } 879246713Skib } else { 880246713Skib if (reserve_bounce_pages(dmat, map, 1) != 0) { 881246713Skib /* Queue us for resources */ 882246713Skib STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 883246713Skib mtx_unlock(&bounce_lock); 884246713Skib return (EINPROGRESS); 885246713Skib } 886166063Scognet } 887246713Skib mtx_unlock(&bounce_lock); 888166063Scognet 889166063Scognet return (0); 890166063Scognet} 891166063Scognet 892129198Scognet/* 893246713Skib * Add a single contiguous physical range to the segment list. 894246713Skib */ 895246713Skibstatic int 896246713Skib_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 897246713Skib bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 898246713Skib{ 899246713Skib bus_addr_t baddr, bmask; 900246713Skib int seg; 901246713Skib 902246713Skib /* 903246713Skib * Make sure we don't cross any boundaries. 904246713Skib */ 905246713Skib bmask = ~(dmat->boundary - 1); 906246713Skib if (dmat->boundary > 0) { 907246713Skib baddr = (curaddr + dmat->boundary) & bmask; 908246713Skib if (sgsize > (baddr - curaddr)) 909246713Skib sgsize = (baddr - curaddr); 910246713Skib } 911246713Skib if (dmat->ranges) { 912246713Skib struct arm32_dma_range *dr; 913246713Skib 914246713Skib dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 915246713Skib curaddr); 916246713Skib if (dr == NULL) 917246881Sian return (0); 918246713Skib /* 919246713Skib * In a valid DMA range. Translate the physical 920246713Skib * memory address to an address in the DMA window. 921246713Skib */ 922246713Skib curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 923283366Sandrew 924246713Skib } 925246713Skib 926246713Skib seg = *segp; 927246713Skib /* 928246713Skib * Insert chunk into a segment, coalescing with 929246713Skib * the previous segment if possible. 930246713Skib */ 931246713Skib if (seg >= 0 && 932246713Skib curaddr == segs[seg].ds_addr + segs[seg].ds_len && 933246713Skib (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 934246713Skib (dmat->boundary == 0 || 935289851Sian (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 936246713Skib segs[seg].ds_len += sgsize; 937246713Skib } else { 938246713Skib if (++seg >= dmat->nsegments) 939246881Sian return (0); 940246713Skib segs[seg].ds_addr = curaddr; 941246713Skib segs[seg].ds_len = sgsize; 942246713Skib } 943246713Skib *segp = seg; 944246881Sian return (sgsize); 945246713Skib} 946246713Skib 947246713Skib/* 948246713Skib * Utility function to load a physical buffer. segp contains 949246713Skib * the starting segment on entrace, and the ending segment on exit. 950246713Skib */ 951246713Skibint 952246713Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 953246713Skib bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 954246713Skib{ 955289675Sjah struct sync_list *sl; 956246713Skib bus_size_t sgsize; 957246713Skib bus_addr_t curaddr; 958289675Sjah bus_addr_t sl_end = 0; 959246713Skib int error; 960246713Skib 961246713Skib if (segs == NULL) 962246713Skib segs = dmat->segments; 963246713Skib 964246713Skib if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 965246713Skib _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 966246713Skib if (map->pagesneeded != 0) { 967246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 968246713Skib if (error) 969246713Skib return (error); 970246713Skib } 971246713Skib } 972246713Skib 973289675Sjah sl = map->slist + map->sync_count - 1; 974289675Sjah 975246713Skib while (buflen > 0) { 976246713Skib curaddr = buf; 977246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 978246713Skib if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 979246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 980289675Sjah sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 981246713Skib curaddr = add_bounce_page(dmat, map, 0, curaddr, 982246713Skib sgsize); 983289675Sjah } else { 984289675Sjah if (map->sync_count > 0) 985289675Sjah sl_end = VM_PAGE_TO_PHYS(sl->pages) + 986289675Sjah sl->dataoffs + sl->datacount; 987289675Sjah 988289675Sjah if (map->sync_count == 0 || curaddr != sl_end) { 989289675Sjah if (++map->sync_count > dmat->nsegments) 990289675Sjah break; 991289675Sjah sl++; 992289675Sjah sl->vaddr = 0; 993289675Sjah sl->datacount = sgsize; 994289675Sjah sl->pages = PHYS_TO_VM_PAGE(curaddr); 995289675Sjah sl->dataoffs = curaddr & PAGE_MASK; 996289675Sjah } else 997289675Sjah sl->datacount += sgsize; 998246713Skib } 999246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1000246713Skib segp); 1001246713Skib if (sgsize == 0) 1002246713Skib break; 1003246713Skib buf += sgsize; 1004246713Skib buflen -= sgsize; 1005246713Skib } 1006246713Skib 1007246713Skib /* 1008246713Skib * Did we fit? 1009246713Skib */ 1010246713Skib if (buflen != 0) { 1011246713Skib _bus_dmamap_unload(dmat, map); 1012246713Skib return (EFBIG); /* XXX better return value here? */ 1013246713Skib } 1014246713Skib return (0); 1015246713Skib} 1016257228Skib 1017257228Skibint 1018257228Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1019257228Skib struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1020257228Skib bus_dma_segment_t *segs, int *segp) 1021257228Skib{ 1022257228Skib 1023257228Skib return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1024257228Skib segs, segp)); 1025257228Skib} 1026257228Skib 1027246713Skib/* 1028246713Skib * Utility function to load a linear buffer. segp contains 1029129198Scognet * the starting segment on entrance, and the ending segment on exit. 1030129198Scognet */ 1031246713Skibint 1032246713Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1033246713Skib bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1034246713Skib int *segp) 1035129198Scognet{ 1036129198Scognet bus_size_t sgsize; 1037246713Skib bus_addr_t curaddr; 1038289675Sjah bus_addr_t sl_pend = 0; 1039246713Skib struct sync_list *sl; 1040289675Sjah vm_offset_t kvaddr; 1041129198Scognet vm_offset_t vaddr = (vm_offset_t)buf; 1042289675Sjah vm_offset_t sl_vend = 0; 1043129198Scognet int error = 0; 1044129198Scognet 1045246713Skib if (segs == NULL) 1046246713Skib segs = dmat->segments; 1047246713Skib if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1048246713Skib map->flags |= DMAMAP_CACHE_ALIGNED; 1049129198Scognet 1050166063Scognet if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1051246713Skib _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1052246713Skib if (map->pagesneeded != 0) { 1053246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 1054246713Skib if (error) 1055246713Skib return (error); 1056246713Skib } 1057166063Scognet } 1058140313Scognet CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1059140313Scognet "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1060140313Scognet 1061289675Sjah sl = map->slist + map->sync_count - 1; 1062289675Sjah 1063246713Skib while (buflen > 0) { 1064129198Scognet /* 1065129198Scognet * Get the physical address for this segment. 1066129198Scognet */ 1067246713Skib if (__predict_true(pmap == kernel_pmap)) { 1068246158Skib curaddr = pmap_kextract(vaddr); 1069289675Sjah kvaddr = vaddr; 1070129198Scognet } else { 1071129198Scognet curaddr = pmap_extract(pmap, vaddr); 1072135644Scognet map->flags &= ~DMAMAP_COHERENT; 1073289675Sjah kvaddr = 0; 1074129198Scognet } 1075129198Scognet 1076129198Scognet /* 1077129198Scognet * Compute the segment size, and adjust counts. 1078129198Scognet */ 1079289675Sjah sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1080170086Syongari if (sgsize > dmat->maxsegsz) 1081170086Syongari sgsize = dmat->maxsegsz; 1082129198Scognet if (buflen < sgsize) 1083129198Scognet sgsize = buflen; 1084129198Scognet 1085166063Scognet if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1086246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1087289675Sjah curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1088246713Skib sgsize); 1089137760Scognet } else { 1090289675Sjah if (map->sync_count > 0) { 1091289675Sjah sl_pend = VM_PAGE_TO_PHYS(sl->pages) + 1092289675Sjah sl->dataoffs + sl->datacount; 1093289675Sjah sl_vend = sl->vaddr + sl->datacount; 1094289675Sjah } 1095289675Sjah 1096246713Skib if (map->sync_count == 0 || 1097289675Sjah (kvaddr != 0 && kvaddr != sl_vend) || 1098289675Sjah (kvaddr == 0 && curaddr != sl_pend)) { 1099289675Sjah 1100246713Skib if (++map->sync_count > dmat->nsegments) 1101246713Skib goto cleanup; 1102246713Skib sl++; 1103289675Sjah sl->vaddr = kvaddr; 1104246713Skib sl->datacount = sgsize; 1105289675Sjah sl->pages = PHYS_TO_VM_PAGE(curaddr); 1106289675Sjah sl->dataoffs = curaddr & PAGE_MASK; 1107246713Skib } else 1108246713Skib sl->datacount += sgsize; 1109129198Scognet } 1110246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1111246713Skib segp); 1112246713Skib if (sgsize == 0) 1113135644Scognet break; 1114129198Scognet vaddr += sgsize; 1115129198Scognet buflen -= sgsize; 1116129198Scognet } 1117129198Scognet 1118246713Skibcleanup: 1119129198Scognet /* 1120129198Scognet * Did we fit? 1121129198Scognet */ 1122246713Skib if (buflen != 0) { 1123246713Skib _bus_dmamap_unload(dmat, map); 1124246713Skib return (EFBIG); /* XXX better return value here? */ 1125246713Skib } 1126246713Skib return (0); 1127129198Scognet} 1128129198Scognet 1129246713Skibvoid 1130289851Sian__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, 1131289851Sian bus_dmamap_callback_t *callback, void *callback_arg) 1132140682Scognet{ 1133140682Scognet 1134143671Sjmg KASSERT(dmat != NULL, ("dmatag is NULL")); 1135143671Sjmg KASSERT(map != NULL, ("dmamap is NULL")); 1136246713Skib map->mem = *mem; 1137166063Scognet map->callback = callback; 1138166063Scognet map->callback_arg = callback_arg; 1139140682Scognet} 1140140682Scognet 1141246713Skibbus_dma_segment_t * 1142246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1143289851Sian bus_dma_segment_t *segs, int nsegs, int error) 1144129198Scognet{ 1145129198Scognet 1146246713Skib if (segs == NULL) 1147246713Skib segs = dmat->segments; 1148246713Skib return (segs); 1149129198Scognet} 1150129198Scognet 1151129198Scognet/* 1152135644Scognet * Release the mapping held by map. 1153129198Scognet */ 1154129198Scognetvoid 1155143655Sjmg_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1156150893Scognet{ 1157166063Scognet struct bounce_page *bpage; 1158166063Scognet 1159166063Scognet while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1160166063Scognet STAILQ_REMOVE_HEAD(&map->bpages, links); 1161166063Scognet free_bounce_page(dmat, bpage); 1162166063Scognet } 1163246713Skib map->sync_count = 0; 1164129198Scognet} 1165129198Scognet 1166169761Scognetstatic void 1167246713Skibbus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1168246713Skib int bufaligned) 1169135644Scognet{ 1170166063Scognet char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1171234561Smarius register_t s; 1172236991Simp int partial; 1173135644Scognet 1174171890Scognet if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1175246713Skib cpu_dcache_wb_range(buf, len); 1176246713Skib cpu_l2cache_wb_range(buf, len); 1177171623Scognet } 1178244471Scognet 1179244471Scognet /* 1180244471Scognet * If the caller promises the buffer is properly aligned to a cache line 1181244471Scognet * (even if the call parms make it look like it isn't) we can avoid 1182244471Scognet * attempting to preserve the non-DMA part of the cache line in the 1183244471Scognet * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1184244471Scognet * 1185244471Scognet * This covers the case of mbufs, where we know how they're aligned and 1186244471Scognet * know the CPU doesn't touch the header in front of the DMA data area 1187244471Scognet * during the IO, but it may have touched it right before invoking the 1188244471Scognet * sync, so a PREREAD writeback is required. 1189244471Scognet * 1190244471Scognet * It also handles buffers we created in bus_dmamem_alloc(), which are 1191244471Scognet * always aligned and padded to cache line size even if the IO length 1192244471Scognet * isn't a multiple of cache line size. In this case the PREREAD 1193244471Scognet * writeback probably isn't required, but it's harmless. 1194244471Scognet */ 1195234561Smarius partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1196244471Scognet 1197171623Scognet if (op & BUS_DMASYNC_PREREAD) { 1198234561Smarius if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1199246713Skib cpu_dcache_inv_range(buf, len); 1200246713Skib cpu_l2cache_inv_range(buf, len); 1201171890Scognet } else { 1202246713Skib cpu_dcache_wbinv_range(buf, len); 1203246713Skib cpu_l2cache_wbinv_range(buf, len); 1204171890Scognet } 1205171623Scognet } 1206166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1207244471Scognet if (partial && !bufaligned) { 1208234561Smarius s = intr_disable(); 1209246713Skib if (buf & arm_dcache_align_mask) 1210246713Skib memcpy(_tmp_cl, (void *)(buf & 1211234561Smarius ~arm_dcache_align_mask), 1212246713Skib buf & arm_dcache_align_mask); 1213246713Skib if ((buf + len) & arm_dcache_align_mask) 1214236991Simp memcpy(_tmp_clend, 1215246713Skib (void *)(buf + len), 1216246713Skib arm_dcache_align - 1217246713Skib ((buf + len) & arm_dcache_align_mask)); 1218171623Scognet } 1219246713Skib cpu_dcache_inv_range(buf, len); 1220246713Skib cpu_l2cache_inv_range(buf, len); 1221244471Scognet if (partial && !bufaligned) { 1222246713Skib if (buf & arm_dcache_align_mask) 1223246713Skib memcpy((void *)(buf & 1224236991Simp ~arm_dcache_align_mask), _tmp_cl, 1225246713Skib buf & arm_dcache_align_mask); 1226246713Skib if ((buf + len) & arm_dcache_align_mask) 1227246713Skib memcpy((void *)(buf + len), 1228236991Simp _tmp_clend, arm_dcache_align - 1229246713Skib ((buf + len) & arm_dcache_align_mask)); 1230234561Smarius intr_restore(s); 1231234561Smarius } 1232135644Scognet } 1233135644Scognet} 1234135644Scognet 1235166063Scognetstatic void 1236289675Sjahbus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 1237289675Sjah int bufaligned) 1238289675Sjah{ 1239289675Sjah vm_offset_t tempvaddr; 1240289675Sjah vm_page_t curpage; 1241289675Sjah size_t npages; 1242289675Sjah 1243289675Sjah if (sl->vaddr != 0) { 1244289675Sjah bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); 1245289675Sjah return; 1246289675Sjah } 1247289675Sjah 1248289675Sjah tempvaddr = 0; 1249289675Sjah npages = atop(round_page(sl->dataoffs + sl->datacount)); 1250289675Sjah 1251289675Sjah for (curpage = sl->pages; curpage != sl->pages + npages; ++curpage) { 1252289675Sjah /* 1253289675Sjah * If the page is mapped to some other VA that hasn't 1254289675Sjah * been supplied to busdma, then pmap_quick_enter_page() 1255289675Sjah * will find all duplicate mappings and mark them 1256289675Sjah * uncacheable. 1257289675Sjah * That will also do any necessary wb/inv. Otherwise, 1258289675Sjah * if the page is truly unmapped, then we don't actually 1259289675Sjah * need to do cache maintenance. 1260289675Sjah * XXX: May overwrite DMA'ed data in the POSTREAD 1261289675Sjah * case where the CPU has written to a cacheline not 1262289675Sjah * completely covered by the DMA region. 1263289675Sjah */ 1264289675Sjah KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) + 1265289675Sjah ptoa(curpage - sl->pages), 1266289675Sjah ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1267289675Sjah VM_PAGE_TO_PHYS(curpage), VM_PAGE_TO_PHYS(sl->pages) + 1268289675Sjah ptoa(curpage - sl->pages))); 1269289675Sjah tempvaddr = pmap_quick_enter_page(curpage); 1270289675Sjah pmap_quick_remove_page(tempvaddr); 1271289675Sjah } 1272289675Sjah} 1273289675Sjah 1274289675Sjahstatic void 1275166063Scognet_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1276166063Scognet{ 1277166063Scognet struct bounce_page *bpage; 1278289675Sjah vm_offset_t datavaddr, tempvaddr; 1279166063Scognet 1280289675Sjah if ((op & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) == 0) 1281289675Sjah return; 1282289675Sjah 1283166063Scognet STAILQ_FOREACH(bpage, &map->bpages, links) { 1284289675Sjah tempvaddr = 0; 1285289675Sjah datavaddr = bpage->datavaddr; 1286166063Scognet if (op & BUS_DMASYNC_PREWRITE) { 1287289675Sjah if (datavaddr == 0) { 1288289675Sjah tempvaddr = 1289289675Sjah pmap_quick_enter_page(bpage->datapage); 1290289675Sjah datavaddr = tempvaddr | bpage->dataoffs; 1291289675Sjah } 1292289675Sjah bcopy((void *)datavaddr, 1293289675Sjah (void *)bpage->vaddr, bpage->datacount); 1294289675Sjah if (tempvaddr != 0) 1295289675Sjah pmap_quick_remove_page(tempvaddr); 1296257201Sian cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1297257201Sian cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1298187911Sthompsa dmat->bounce_zone->total_bounced++; 1299166063Scognet } 1300166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1301257201Sian cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1302257201Sian cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1303289675Sjah if (datavaddr == 0) { 1304289675Sjah tempvaddr = 1305289675Sjah pmap_quick_enter_page(bpage->datapage); 1306289675Sjah datavaddr = tempvaddr | bpage->dataoffs; 1307289675Sjah } 1308289675Sjah bcopy((void *)bpage->vaddr, 1309289675Sjah (void *)datavaddr, bpage->datacount); 1310289675Sjah if (tempvaddr != 0) 1311289675Sjah pmap_quick_remove_page(tempvaddr); 1312187911Sthompsa dmat->bounce_zone->total_bounced++; 1313166063Scognet } 1314166063Scognet } 1315166063Scognet} 1316166063Scognet 1317129198Scognetvoid 1318143655Sjmg_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1319129198Scognet{ 1320246713Skib struct sync_list *sl, *end; 1321244471Scognet int bufaligned; 1322244471Scognet 1323159107Scognet if (op == BUS_DMASYNC_POSTWRITE) 1324129198Scognet return; 1325244471Scognet if (map->flags & DMAMAP_COHERENT) 1326244471Scognet goto drain; 1327166063Scognet if (STAILQ_FIRST(&map->bpages)) 1328166063Scognet _bus_dmamap_sync_bp(dmat, map, op); 1329143294Smux CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1330244471Scognet bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1331246713Skib if (map->sync_count) { 1332246713Skib end = &map->slist[map->sync_count]; 1333246713Skib for (sl = &map->slist[0]; sl != end; sl++) 1334289675Sjah bus_dmamap_sync_sl(sl, op, bufaligned); 1335129198Scognet } 1336244471Scognet 1337244471Scognetdrain: 1338244471Scognet 1339135644Scognet cpu_drain_writebuf(); 1340129198Scognet} 1341166063Scognet 1342166063Scognetstatic void 1343166063Scognetinit_bounce_pages(void *dummy __unused) 1344166063Scognet{ 1345166063Scognet 1346166063Scognet total_bpages = 0; 1347166063Scognet STAILQ_INIT(&bounce_zone_list); 1348166063Scognet STAILQ_INIT(&bounce_map_waitinglist); 1349166063Scognet STAILQ_INIT(&bounce_map_callbacklist); 1350166063Scognet mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1351166063Scognet} 1352166063ScognetSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1353166063Scognet 1354166063Scognetstatic struct sysctl_ctx_list * 1355166063Scognetbusdma_sysctl_tree(struct bounce_zone *bz) 1356166063Scognet{ 1357289851Sian 1358166063Scognet return (&bz->sysctl_tree); 1359166063Scognet} 1360166063Scognet 1361166063Scognetstatic struct sysctl_oid * 1362166063Scognetbusdma_sysctl_tree_top(struct bounce_zone *bz) 1363166063Scognet{ 1364289851Sian 1365166063Scognet return (bz->sysctl_tree_top); 1366166063Scognet} 1367166063Scognet 1368166063Scognetstatic int 1369166063Scognetalloc_bounce_zone(bus_dma_tag_t dmat) 1370166063Scognet{ 1371166063Scognet struct bounce_zone *bz; 1372166063Scognet 1373166063Scognet /* Check to see if we already have a suitable zone */ 1374166063Scognet STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1375289851Sian if ((dmat->alignment <= bz->alignment) && 1376289851Sian (dmat->lowaddr >= bz->lowaddr)) { 1377166063Scognet dmat->bounce_zone = bz; 1378166063Scognet return (0); 1379166063Scognet } 1380166063Scognet } 1381166063Scognet 1382166063Scognet if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1383166063Scognet M_NOWAIT | M_ZERO)) == NULL) 1384166063Scognet return (ENOMEM); 1385166063Scognet 1386166063Scognet STAILQ_INIT(&bz->bounce_page_list); 1387166063Scognet bz->free_bpages = 0; 1388166063Scognet bz->reserved_bpages = 0; 1389166063Scognet bz->active_bpages = 0; 1390166063Scognet bz->lowaddr = dmat->lowaddr; 1391191438Sjhb bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1392188403Scognet bz->map_count = 0; 1393166063Scognet snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1394166063Scognet busdma_zonecount++; 1395166063Scognet snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1396166063Scognet STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1397166063Scognet dmat->bounce_zone = bz; 1398166063Scognet 1399166063Scognet sysctl_ctx_init(&bz->sysctl_tree); 1400166063Scognet bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1401166063Scognet SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1402166063Scognet CTLFLAG_RD, 0, ""); 1403166063Scognet if (bz->sysctl_tree_top == NULL) { 1404166063Scognet sysctl_ctx_free(&bz->sysctl_tree); 1405166063Scognet return (0); /* XXX error code? */ 1406166063Scognet } 1407166063Scognet 1408166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1409166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1410166063Scognet "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1411166063Scognet "Total bounce pages"); 1412166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1413166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1414166063Scognet "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1415166063Scognet "Free bounce pages"); 1416166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1417166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1418166063Scognet "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1419166063Scognet "Reserved bounce pages"); 1420166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1421166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1422166063Scognet "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1423166063Scognet "Active bounce pages"); 1424166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1425166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1426166063Scognet "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1427289851Sian "Total bounce requests (pages bounced)"); 1428166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1429166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1430166063Scognet "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1431166063Scognet "Total bounce requests that were deferred"); 1432166063Scognet SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1433166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1434166063Scognet "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1435273377Shselasky SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1436166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1437273377Shselasky "alignment", CTLFLAG_RD, &bz->alignment, ""); 1438166063Scognet 1439166063Scognet return (0); 1440166063Scognet} 1441166063Scognet 1442166063Scognetstatic int 1443166063Scognetalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1444166063Scognet{ 1445166063Scognet struct bounce_zone *bz; 1446166063Scognet int count; 1447166063Scognet 1448166063Scognet bz = dmat->bounce_zone; 1449166063Scognet count = 0; 1450166063Scognet while (numpages > 0) { 1451166063Scognet struct bounce_page *bpage; 1452166063Scognet 1453166063Scognet bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1454289851Sian M_NOWAIT | M_ZERO); 1455166063Scognet 1456166063Scognet if (bpage == NULL) 1457166063Scognet break; 1458166063Scognet bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1459289851Sian M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1460166063Scognet if (bpage->vaddr == 0) { 1461166063Scognet free(bpage, M_DEVBUF); 1462166063Scognet break; 1463166063Scognet } 1464166063Scognet bpage->busaddr = pmap_kextract(bpage->vaddr); 1465166063Scognet mtx_lock(&bounce_lock); 1466166063Scognet STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1467166063Scognet total_bpages++; 1468166063Scognet bz->total_bpages++; 1469166063Scognet bz->free_bpages++; 1470166063Scognet mtx_unlock(&bounce_lock); 1471166063Scognet count++; 1472166063Scognet numpages--; 1473166063Scognet } 1474166063Scognet return (count); 1475166063Scognet} 1476166063Scognet 1477166063Scognetstatic int 1478166063Scognetreserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1479166063Scognet{ 1480166063Scognet struct bounce_zone *bz; 1481166063Scognet int pages; 1482166063Scognet 1483166063Scognet mtx_assert(&bounce_lock, MA_OWNED); 1484166063Scognet bz = dmat->bounce_zone; 1485166063Scognet pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1486166063Scognet if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1487166063Scognet return (map->pagesneeded - (map->pagesreserved + pages)); 1488166063Scognet bz->free_bpages -= pages; 1489166063Scognet bz->reserved_bpages += pages; 1490166063Scognet map->pagesreserved += pages; 1491166063Scognet pages = map->pagesneeded - map->pagesreserved; 1492166063Scognet 1493166063Scognet return (pages); 1494166063Scognet} 1495166063Scognet 1496166063Scognetstatic bus_addr_t 1497166063Scognetadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1498289851Sian bus_addr_t addr, bus_size_t size) 1499166063Scognet{ 1500166063Scognet struct bounce_zone *bz; 1501166063Scognet struct bounce_page *bpage; 1502166063Scognet 1503166063Scognet KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1504170406Scognet KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1505166063Scognet 1506166063Scognet bz = dmat->bounce_zone; 1507166063Scognet if (map->pagesneeded == 0) 1508166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1509166063Scognet map->pagesneeded--; 1510166063Scognet 1511166063Scognet if (map->pagesreserved == 0) 1512166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1513166063Scognet map->pagesreserved--; 1514166063Scognet 1515166063Scognet mtx_lock(&bounce_lock); 1516166063Scognet bpage = STAILQ_FIRST(&bz->bounce_page_list); 1517166063Scognet if (bpage == NULL) 1518166063Scognet panic("add_bounce_page: free page list is empty"); 1519166063Scognet 1520166063Scognet STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1521166063Scognet bz->reserved_bpages--; 1522166063Scognet bz->active_bpages++; 1523166063Scognet mtx_unlock(&bounce_lock); 1524166063Scognet 1525188350Simp if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1526191201Sjhb /* Page offset needs to be preserved. */ 1527282120Shselasky bpage->vaddr |= addr & PAGE_MASK; 1528282120Shselasky bpage->busaddr |= addr & PAGE_MASK; 1529188350Simp } 1530166063Scognet bpage->datavaddr = vaddr; 1531289675Sjah bpage->datapage = PHYS_TO_VM_PAGE(addr); 1532289675Sjah bpage->dataoffs = addr & PAGE_MASK; 1533166063Scognet bpage->datacount = size; 1534166063Scognet STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1535166063Scognet return (bpage->busaddr); 1536166063Scognet} 1537166063Scognet 1538166063Scognetstatic void 1539166063Scognetfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1540166063Scognet{ 1541166063Scognet struct bus_dmamap *map; 1542166063Scognet struct bounce_zone *bz; 1543166063Scognet 1544166063Scognet bz = dmat->bounce_zone; 1545166063Scognet bpage->datavaddr = 0; 1546166063Scognet bpage->datacount = 0; 1547191201Sjhb if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1548191201Sjhb /* 1549191201Sjhb * Reset the bounce page to start at offset 0. Other uses 1550191201Sjhb * of this bounce page may need to store a full page of 1551191201Sjhb * data and/or assume it starts on a page boundary. 1552191201Sjhb */ 1553191201Sjhb bpage->vaddr &= ~PAGE_MASK; 1554191201Sjhb bpage->busaddr &= ~PAGE_MASK; 1555191201Sjhb } 1556166063Scognet 1557166063Scognet mtx_lock(&bounce_lock); 1558166063Scognet STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1559166063Scognet bz->free_bpages++; 1560166063Scognet bz->active_bpages--; 1561166063Scognet if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1562166063Scognet if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1563166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1564166063Scognet STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1565289851Sian map, links); 1566166063Scognet busdma_swi_pending = 1; 1567166063Scognet bz->total_deferred++; 1568166063Scognet swi_sched(vm_ih, 0); 1569166063Scognet } 1570166063Scognet } 1571166063Scognet mtx_unlock(&bounce_lock); 1572166063Scognet} 1573166063Scognet 1574166063Scognetvoid 1575166063Scognetbusdma_swi(void) 1576166063Scognet{ 1577166063Scognet bus_dma_tag_t dmat; 1578166063Scognet struct bus_dmamap *map; 1579166063Scognet 1580166063Scognet mtx_lock(&bounce_lock); 1581166063Scognet while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1582166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1583166063Scognet mtx_unlock(&bounce_lock); 1584166063Scognet dmat = map->dmat; 1585289851Sian dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1586289851Sian bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1587289851Sian map->callback_arg, BUS_DMA_WAITOK); 1588289851Sian dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1589166063Scognet mtx_lock(&bounce_lock); 1590166063Scognet } 1591166063Scognet mtx_unlock(&bounce_lock); 1592166063Scognet} 1593