busdma_machdep-v4.c revision 273377
1139735Simp/*- 2244471Scognet * Copyright (c) 2012 Ian Lepore 3129198Scognet * Copyright (c) 2004 Olivier Houchard 4129198Scognet * Copyright (c) 2002 Peter Grehan 5129198Scognet * Copyright (c) 1997, 1998 Justin T. Gibbs. 6129198Scognet * All rights reserved. 7129198Scognet * 8129198Scognet * Redistribution and use in source and binary forms, with or without 9129198Scognet * modification, are permitted provided that the following conditions 10129198Scognet * are met: 11129198Scognet * 1. Redistributions of source code must retain the above copyright 12129198Scognet * notice, this list of conditions, and the following disclaimer, 13129198Scognet * without modification, immediately at the beginning of the file. 14129198Scognet * 2. The name of the author may not be used to endorse or promote products 15129198Scognet * derived from this software without specific prior written permission. 16129198Scognet * 17129198Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18129198Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19129198Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20129198Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21129198Scognet * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22129198Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23129198Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24129198Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25129198Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26129198Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27129198Scognet * SUCH DAMAGE. 28129198Scognet * 29129198Scognet * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30129198Scognet */ 31129198Scognet 32129198Scognet#include <sys/cdefs.h> 33129198Scognet__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 273377 2014-10-21 07:31:21Z hselasky $"); 34129198Scognet 35129198Scognet/* 36244471Scognet * ARM bus dma support routines. 37244471Scognet * 38244471Scognet * XXX Things to investigate / fix some day... 39244471Scognet * - What is the earliest that this API can be called? Could there be any 40244471Scognet * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41244471Scognet * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42244471Scognet * bus_dmamap_load() function. This code has historically (and still does) 43244471Scognet * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44244471Scognet * error checking because some resource management calls would become WAITOK 45244471Scognet * and thus "cannot fail." 46244471Scognet * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47244471Scognet * creation time, and the result stored in the tag. 48244471Scognet * - It should be possible to take some shortcuts when mapping a buffer we know 49244471Scognet * came from the uma(9) allocators based on what we know about such buffers 50244471Scognet * (aligned, contiguous, etc). 51244471Scognet * - The allocation of bounce pages could probably be cleaned up, then we could 52244471Scognet * retire arm_remap_nocache(). 53129198Scognet */ 54129198Scognet 55129198Scognet#define _ARM32_BUS_DMA_PRIVATE 56129198Scognet#include <sys/param.h> 57129198Scognet#include <sys/systm.h> 58129198Scognet#include <sys/malloc.h> 59129198Scognet#include <sys/bus.h> 60244471Scognet#include <sys/busdma_bufalloc.h> 61129198Scognet#include <sys/interrupt.h> 62129198Scognet#include <sys/lock.h> 63129198Scognet#include <sys/proc.h> 64246713Skib#include <sys/memdesc.h> 65129198Scognet#include <sys/mutex.h> 66140310Scognet#include <sys/ktr.h> 67146597Scognet#include <sys/kernel.h> 68166063Scognet#include <sys/sysctl.h> 69246713Skib#include <sys/uio.h> 70129198Scognet 71244471Scognet#include <vm/uma.h> 72129198Scognet#include <vm/vm.h> 73244471Scognet#include <vm/vm_extern.h> 74244471Scognet#include <vm/vm_kern.h> 75129198Scognet#include <vm/vm_page.h> 76129198Scognet#include <vm/vm_map.h> 77129198Scognet 78129198Scognet#include <machine/atomic.h> 79129198Scognet#include <machine/bus.h> 80129198Scognet#include <machine/cpufunc.h> 81166063Scognet#include <machine/md_var.h> 82129198Scognet 83166063Scognet#define MAX_BPAGES 64 84166063Scognet#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85166063Scognet#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86166063Scognet 87166063Scognetstruct bounce_zone; 88166063Scognet 89129198Scognetstruct bus_dma_tag { 90129198Scognet bus_dma_tag_t parent; 91129198Scognet bus_size_t alignment; 92232356Sjhb bus_addr_t boundary; 93129198Scognet bus_addr_t lowaddr; 94129198Scognet bus_addr_t highaddr; 95129198Scognet bus_dma_filter_t *filter; 96129198Scognet void *filterarg; 97129198Scognet bus_size_t maxsize; 98129198Scognet u_int nsegments; 99129198Scognet bus_size_t maxsegsz; 100129198Scognet int flags; 101129198Scognet int ref_count; 102129198Scognet int map_count; 103129198Scognet bus_dma_lock_t *lockfunc; 104129198Scognet void *lockfuncarg; 105129198Scognet /* 106129198Scognet * DMA range for this tag. If the page doesn't fall within 107129198Scognet * one of these ranges, an error is returned. The caller 108129198Scognet * may then decide what to do with the transfer. If the 109129198Scognet * range pointer is NULL, it is ignored. 110129198Scognet */ 111129198Scognet struct arm32_dma_range *ranges; 112129198Scognet int _nranges; 113166063Scognet struct bounce_zone *bounce_zone; 114244471Scognet /* 115244471Scognet * Most tags need one or two segments, and can use the local tagsegs 116244471Scognet * array. For tags with a larger limit, we'll allocate a bigger array 117244471Scognet * on first use. 118244471Scognet */ 119244471Scognet bus_dma_segment_t *segments; 120244471Scognet bus_dma_segment_t tagsegs[2]; 121129198Scognet}; 122129198Scognet 123166063Scognetstruct bounce_page { 124166063Scognet vm_offset_t vaddr; /* kva of bounce buffer */ 125166063Scognet bus_addr_t busaddr; /* Physical address */ 126166063Scognet vm_offset_t datavaddr; /* kva of client data */ 127246713Skib bus_addr_t dataaddr; /* client physical address */ 128166063Scognet bus_size_t datacount; /* client data count */ 129166063Scognet STAILQ_ENTRY(bounce_page) links; 130166063Scognet}; 131166063Scognet 132246713Skibstruct sync_list { 133246713Skib vm_offset_t vaddr; /* kva of bounce buffer */ 134246713Skib bus_addr_t busaddr; /* Physical address */ 135246713Skib bus_size_t datacount; /* client data count */ 136246713Skib}; 137246713Skib 138166063Scognetint busdma_swi_pending; 139166063Scognet 140166063Scognetstruct bounce_zone { 141166063Scognet STAILQ_ENTRY(bounce_zone) links; 142166063Scognet STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 143166063Scognet int total_bpages; 144166063Scognet int free_bpages; 145166063Scognet int reserved_bpages; 146166063Scognet int active_bpages; 147166063Scognet int total_bounced; 148166063Scognet int total_deferred; 149188403Scognet int map_count; 150166063Scognet bus_size_t alignment; 151166063Scognet bus_addr_t lowaddr; 152166063Scognet char zoneid[8]; 153166063Scognet char lowaddrid[20]; 154166063Scognet struct sysctl_ctx_list sysctl_tree; 155166063Scognet struct sysctl_oid *sysctl_tree_top; 156166063Scognet}; 157166063Scognet 158166063Scognetstatic struct mtx bounce_lock; 159166063Scognetstatic int total_bpages; 160166063Scognetstatic int busdma_zonecount; 161166063Scognetstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 162166063Scognet 163227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 164166063ScognetSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 165166063Scognet "Total bounce pages"); 166166063Scognet 167246713Skib#define DMAMAP_COHERENT 0x8 168244471Scognet#define DMAMAP_CACHE_ALIGNED 0x10 169246713Skib 170129198Scognetstruct bus_dmamap { 171166063Scognet struct bp_list bpages; 172166063Scognet int pagesneeded; 173166063Scognet int pagesreserved; 174135644Scognet bus_dma_tag_t dmat; 175246713Skib struct memdesc mem; 176135644Scognet int flags; 177166063Scognet STAILQ_ENTRY(bus_dmamap) links; 178166063Scognet bus_dmamap_callback_t *callback; 179166063Scognet void *callback_arg; 180246713Skib int sync_count; 181246713Skib struct sync_list *slist; 182129198Scognet}; 183129198Scognet 184166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 185166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 186166063Scognet 187146597Scognetstatic struct mtx busdma_mtx; 188146597Scognet 189146597ScognetMTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 190146597Scognet 191166063Scognetstatic void init_bounce_pages(void *dummy); 192166063Scognetstatic int alloc_bounce_zone(bus_dma_tag_t dmat); 193166063Scognetstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 194166063Scognetstatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 195166063Scognet int commit); 196166063Scognetstatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 197246713Skib vm_offset_t vaddr, bus_addr_t addr, 198246713Skib bus_size_t size); 199166063Scognetstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200166063Scognet 201166063Scognet/* Default tag, as most drivers provide no parent tag. */ 202166063Scognetbus_dma_tag_t arm_root_dma_tag; 203166063Scognet 204244473Scognet/* 205244473Scognet * ---------------------------------------------------------------------------- 206244473Scognet * Begin block of code useful to transplant to other implementations. 207244473Scognet */ 208244471Scognet 209244471Scognetstatic uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 210244471Scognet 211244471Scognetstatic busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 212244471Scognetstatic busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 213244471Scognet 214166063Scognet/* 215244471Scognet * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 216244471Scognet * It'll need platform-specific changes if this code is copied. 217244471Scognet */ 218244471Scognetstatic int 219244471Scognetdmamap_ctor(void *mem, int size, void *arg, int flags) 220244471Scognet{ 221244471Scognet bus_dmamap_t map; 222244471Scognet bus_dma_tag_t dmat; 223244471Scognet 224244471Scognet map = (bus_dmamap_t)mem; 225244471Scognet dmat = (bus_dma_tag_t)arg; 226244471Scognet 227244471Scognet dmat->map_count++; 228244471Scognet 229244471Scognet map->dmat = dmat; 230244471Scognet map->flags = 0; 231244471Scognet STAILQ_INIT(&map->bpages); 232244471Scognet 233244471Scognet return (0); 234244471Scognet} 235244471Scognet 236244471Scognet/* 237244471Scognet * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 238244471Scognet * It may need platform-specific changes if this code is copied . 239244471Scognet */ 240244471Scognetstatic void 241244471Scognetdmamap_dtor(void *mem, int size, void *arg) 242244471Scognet{ 243244471Scognet bus_dmamap_t map; 244244471Scognet 245244471Scognet map = (bus_dmamap_t)mem; 246244471Scognet 247244471Scognet map->dmat->map_count--; 248244471Scognet} 249244471Scognet 250244471Scognetstatic void 251244471Scognetbusdma_init(void *dummy) 252244471Scognet{ 253244471Scognet 254244471Scognet /* Create a cache of maps for bus_dmamap_create(). */ 255244471Scognet dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 256244471Scognet dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 257244471Scognet 258244471Scognet /* Create a cache of buffers in standard (cacheable) memory. */ 259244471Scognet standard_allocator = busdma_bufalloc_create("buffer", 260244471Scognet arm_dcache_align, /* minimum_alignment */ 261244471Scognet NULL, /* uma_alloc func */ 262244471Scognet NULL, /* uma_free func */ 263244471Scognet 0); /* uma_zcreate_flags */ 264244471Scognet 265244471Scognet /* 266244471Scognet * Create a cache of buffers in uncacheable memory, to implement the 267244471Scognet * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 268244471Scognet */ 269244471Scognet coherent_allocator = busdma_bufalloc_create("coherent", 270244471Scognet arm_dcache_align, /* minimum_alignment */ 271244471Scognet busdma_bufalloc_alloc_uncacheable, 272244471Scognet busdma_bufalloc_free_uncacheable, 273244471Scognet 0); /* uma_zcreate_flags */ 274244471Scognet} 275244471Scognet 276244471Scognet/* 277244471Scognet * This init historically used SI_SUB_VM, but now the init code requires 278244471Scognet * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 279267992Shselasky * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 280267992Shselasky * SI_SUB_KMEM and SI_ORDER_FOURTH. 281244471Scognet */ 282267992ShselaskySYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 283244471Scognet 284244473Scognet/* 285244473Scognet * End block of code useful to transplant to other implementations. 286244473Scognet * ---------------------------------------------------------------------------- 287244473Scognet */ 288244471Scognet 289244471Scognet/* 290166063Scognet * Return true if a match is made. 291166063Scognet * 292166063Scognet * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 293166063Scognet * 294166063Scognet * If paddr is within the bounds of the dma tag then call the filter callback 295166063Scognet * to check for a match, if there is no filter callback then assume a match. 296166063Scognet */ 297166063Scognetstatic int 298166063Scognetrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 299166063Scognet{ 300166063Scognet int retval; 301166063Scognet 302166063Scognet retval = 0; 303166063Scognet 304166063Scognet do { 305166063Scognet if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 306166063Scognet || ((paddr & (dmat->alignment - 1)) != 0)) 307166063Scognet && (dmat->filter == NULL 308166063Scognet || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 309166063Scognet retval = 1; 310166063Scognet 311166063Scognet dmat = dmat->parent; 312166063Scognet } while (retval == 0 && dmat != NULL); 313166063Scognet return (retval); 314166063Scognet} 315166063Scognet 316129198Scognet/* 317244471Scognet * This routine checks the exclusion zone constraints from a tag against the 318244471Scognet * physical RAM available on the machine. If a tag specifies an exclusion zone 319244471Scognet * but there's no RAM in that zone, then we avoid allocating resources to bounce 320244471Scognet * a request, and we can use any memory allocator (as opposed to needing 321244471Scognet * kmem_alloc_contig() just because it can allocate pages in an address range). 322244471Scognet * 323244471Scognet * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 324244471Scognet * same value on 32-bit architectures) as their lowaddr constraint, and we can't 325244471Scognet * possibly have RAM at an address higher than the highest address we can 326244471Scognet * express, so we take a fast out. 327129198Scognet */ 328137758Scognetstatic __inline int 329166063Scognet_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 330166063Scognet{ 331166063Scognet int i; 332244471Scognet 333244471Scognet if (lowaddr >= BUS_SPACE_MAXADDR) 334244471Scognet return (0); 335244471Scognet 336166063Scognet for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 337166063Scognet if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 338236991Simp || (lowaddr < phys_avail[i] && 339166063Scognet highaddr > phys_avail[i])) 340166063Scognet return (1); 341166063Scognet } 342166063Scognet return (0); 343166063Scognet} 344166063Scognet 345129198Scognetstatic __inline struct arm32_dma_range * 346129198Scognet_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 347129198Scognet bus_addr_t curaddr) 348129198Scognet{ 349129198Scognet struct arm32_dma_range *dr; 350129198Scognet int i; 351129198Scognet 352129198Scognet for (i = 0, dr = ranges; i < nranges; i++, dr++) { 353129198Scognet if (curaddr >= dr->dr_sysbase && 354129198Scognet round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 355129198Scognet return (dr); 356129198Scognet } 357129198Scognet 358129198Scognet return (NULL); 359129198Scognet} 360129198Scognet/* 361129198Scognet * Convenience function for manipulating driver locks from busdma (during 362129198Scognet * busdma_swi, for example). Drivers that don't provide their own locks 363129198Scognet * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 364129198Scognet * non-mutex locking scheme don't have to use this at all. 365129198Scognet */ 366129198Scognetvoid 367129198Scognetbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 368129198Scognet{ 369129198Scognet struct mtx *dmtx; 370129198Scognet 371129198Scognet dmtx = (struct mtx *)arg; 372129198Scognet switch (op) { 373129198Scognet case BUS_DMA_LOCK: 374129198Scognet mtx_lock(dmtx); 375129198Scognet break; 376129198Scognet case BUS_DMA_UNLOCK: 377129198Scognet mtx_unlock(dmtx); 378129198Scognet break; 379129198Scognet default: 380129198Scognet panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 381129198Scognet } 382129198Scognet} 383129198Scognet 384129198Scognet/* 385129198Scognet * dflt_lock should never get called. It gets put into the dma tag when 386129198Scognet * lockfunc == NULL, which is only valid if the maps that are associated 387129198Scognet * with the tag are meant to never be defered. 388129198Scognet * XXX Should have a way to identify which driver is responsible here. 389129198Scognet */ 390129198Scognetstatic void 391129198Scognetdflt_lock(void *arg, bus_dma_lock_op_t op) 392129198Scognet{ 393129198Scognet#ifdef INVARIANTS 394129198Scognet panic("driver error: busdma dflt_lock called"); 395129198Scognet#else 396129198Scognet printf("DRIVER_ERROR: busdma dflt_lock called\n"); 397129198Scognet#endif 398129198Scognet} 399129198Scognet 400129198Scognet/* 401129198Scognet * Allocate a device specific dma_tag. 402129198Scognet */ 403135644Scognet#define SEG_NB 1024 404135644Scognet 405129198Scognetint 406129198Scognetbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 407232356Sjhb bus_addr_t boundary, bus_addr_t lowaddr, 408129198Scognet bus_addr_t highaddr, bus_dma_filter_t *filter, 409129198Scognet void *filterarg, bus_size_t maxsize, int nsegments, 410129198Scognet bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 411129198Scognet void *lockfuncarg, bus_dma_tag_t *dmat) 412129198Scognet{ 413129198Scognet bus_dma_tag_t newtag; 414129198Scognet int error = 0; 415129198Scognet /* Return a NULL tag on failure */ 416129198Scognet *dmat = NULL; 417166063Scognet if (!parent) 418166063Scognet parent = arm_root_dma_tag; 419129198Scognet 420129198Scognet newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 421140313Scognet if (newtag == NULL) { 422143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 423143284Smux __func__, newtag, 0, error); 424129198Scognet return (ENOMEM); 425140313Scognet } 426129198Scognet 427129198Scognet newtag->parent = parent; 428244471Scognet newtag->alignment = alignment ? alignment : 1; 429129198Scognet newtag->boundary = boundary; 430129198Scognet newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 431129198Scognet newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 432129198Scognet newtag->filter = filter; 433129198Scognet newtag->filterarg = filterarg; 434129198Scognet newtag->maxsize = maxsize; 435129198Scognet newtag->nsegments = nsegments; 436129198Scognet newtag->maxsegsz = maxsegsz; 437129198Scognet newtag->flags = flags; 438129198Scognet newtag->ref_count = 1; /* Count ourself */ 439129198Scognet newtag->map_count = 0; 440129198Scognet newtag->ranges = bus_dma_get_range(); 441135644Scognet newtag->_nranges = bus_dma_get_range_nb(); 442129198Scognet if (lockfunc != NULL) { 443129198Scognet newtag->lockfunc = lockfunc; 444129198Scognet newtag->lockfuncarg = lockfuncarg; 445129198Scognet } else { 446129198Scognet newtag->lockfunc = dflt_lock; 447129198Scognet newtag->lockfuncarg = NULL; 448129198Scognet } 449244471Scognet /* 450244471Scognet * If all the segments we need fit into the local tagsegs array, set the 451244471Scognet * pointer now. Otherwise NULL the pointer and an array of segments 452244471Scognet * will be allocated later, on first use. We don't pre-allocate now 453244471Scognet * because some tags exist just to pass contraints to children in the 454244471Scognet * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 455244471Scognet * sure don't want to try to allocate an array for that. 456244471Scognet */ 457244471Scognet if (newtag->nsegments <= nitems(newtag->tagsegs)) 458244471Scognet newtag->segments = newtag->tagsegs; 459244471Scognet else 460244471Scognet newtag->segments = NULL; 461244471Scognet /* 462129198Scognet * Take into account any restrictions imposed by our parent tag 463129198Scognet */ 464129198Scognet if (parent != NULL) { 465232356Sjhb newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 466232356Sjhb newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 467134934Sscottl if (newtag->boundary == 0) 468134934Sscottl newtag->boundary = parent->boundary; 469134934Sscottl else if (parent->boundary != 0) 470232356Sjhb newtag->boundary = MIN(parent->boundary, 471134934Sscottl newtag->boundary); 472166063Scognet if ((newtag->filter != NULL) || 473166063Scognet ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 474166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 475129198Scognet if (newtag->filter == NULL) { 476129198Scognet /* 477129198Scognet * Short circuit looking at our parent directly 478129198Scognet * since we have encapsulated all of its information 479129198Scognet */ 480129198Scognet newtag->filter = parent->filter; 481129198Scognet newtag->filterarg = parent->filterarg; 482129198Scognet newtag->parent = parent->parent; 483129198Scognet } 484129198Scognet if (newtag->parent != NULL) 485129198Scognet atomic_add_int(&parent->ref_count, 1); 486129198Scognet } 487166063Scognet if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 488166063Scognet || newtag->alignment > 1) 489166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 490129198Scognet 491166063Scognet if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 492166063Scognet (flags & BUS_DMA_ALLOCNOW) != 0) { 493166063Scognet struct bounce_zone *bz; 494166063Scognet 495166063Scognet /* Must bounce */ 496166063Scognet 497166063Scognet if ((error = alloc_bounce_zone(newtag)) != 0) { 498166063Scognet free(newtag, M_DEVBUF); 499166063Scognet return (error); 500166063Scognet } 501166063Scognet bz = newtag->bounce_zone; 502166063Scognet 503166063Scognet if (ptoa(bz->total_bpages) < maxsize) { 504166063Scognet int pages; 505166063Scognet 506166063Scognet pages = atop(maxsize) - bz->total_bpages; 507166063Scognet 508166063Scognet /* Add pages to our bounce pool */ 509166063Scognet if (alloc_bounce_pages(newtag, pages) < pages) 510166063Scognet error = ENOMEM; 511166063Scognet } 512166063Scognet /* Performed initial allocation */ 513166063Scognet newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 514170502Scognet } else 515170502Scognet newtag->bounce_zone = NULL; 516166063Scognet if (error != 0) 517166063Scognet free(newtag, M_DEVBUF); 518166063Scognet else 519166063Scognet *dmat = newtag; 520143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 521143284Smux __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 522140313Scognet 523129198Scognet return (error); 524129198Scognet} 525129198Scognet 526129198Scognetint 527129198Scognetbus_dma_tag_destroy(bus_dma_tag_t dmat) 528129198Scognet{ 529140680Scognet#ifdef KTR 530140313Scognet bus_dma_tag_t dmat_copy = dmat; 531140680Scognet#endif 532140313Scognet 533129198Scognet if (dmat != NULL) { 534244471Scognet 535129198Scognet if (dmat->map_count != 0) 536129198Scognet return (EBUSY); 537129198Scognet 538129198Scognet while (dmat != NULL) { 539129198Scognet bus_dma_tag_t parent; 540129198Scognet 541129198Scognet parent = dmat->parent; 542129198Scognet atomic_subtract_int(&dmat->ref_count, 1); 543129198Scognet if (dmat->ref_count == 0) { 544244471Scognet if (dmat->segments != NULL && 545244471Scognet dmat->segments != dmat->tagsegs) 546240177Sjhb free(dmat->segments, M_DEVBUF); 547129198Scognet free(dmat, M_DEVBUF); 548129198Scognet /* 549129198Scognet * Last reference count, so 550129198Scognet * release our reference 551129198Scognet * count on our parent. 552129198Scognet */ 553129198Scognet dmat = parent; 554129198Scognet } else 555129198Scognet dmat = NULL; 556129198Scognet } 557129198Scognet } 558143294Smux CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 559140313Scognet 560129198Scognet return (0); 561129198Scognet} 562129198Scognet 563166063Scognet#include <sys/kdb.h> 564129198Scognet/* 565129198Scognet * Allocate a handle for mapping from kva/uva/physical 566129198Scognet * address space into bus device space. 567129198Scognet */ 568129198Scognetint 569129198Scognetbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 570129198Scognet{ 571246713Skib struct sync_list *slist; 572244471Scognet bus_dmamap_t map; 573140313Scognet int error = 0; 574129198Scognet 575246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 576246713Skib if (slist == NULL) 577246713Skib return (ENOMEM); 578246713Skib 579244575Scognet map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 580244471Scognet *mapp = map; 581246713Skib if (map == NULL) { 582246713Skib free(slist, M_DEVBUF); 583244575Scognet return (ENOMEM); 584246713Skib } 585240177Sjhb 586244471Scognet /* 587244471Scognet * If the tag's segments haven't been allocated yet we need to do it 588244471Scognet * now, because we can't sleep for resources at map load time. 589244471Scognet */ 590244575Scognet if (dmat->segments == NULL) { 591244471Scognet dmat->segments = malloc(dmat->nsegments * 592244575Scognet sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 593244575Scognet if (dmat->segments == NULL) { 594246713Skib free(slist, M_DEVBUF); 595244575Scognet uma_zfree(dmamap_zone, map); 596244575Scognet *mapp = NULL; 597244575Scognet return (ENOMEM); 598244575Scognet } 599244575Scognet } 600129198Scognet 601166063Scognet /* 602166063Scognet * Bouncing might be required if the driver asks for an active 603166063Scognet * exclusion region, a data alignment that is stricter than 1, and/or 604166063Scognet * an active address boundary. 605166063Scognet */ 606166063Scognet if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 607166063Scognet 608166063Scognet /* Must bounce */ 609166063Scognet struct bounce_zone *bz; 610166063Scognet int maxpages; 611166063Scognet 612166063Scognet if (dmat->bounce_zone == NULL) { 613166063Scognet if ((error = alloc_bounce_zone(dmat)) != 0) { 614246713Skib free(slist, M_DEVBUF); 615244471Scognet uma_zfree(dmamap_zone, map); 616166063Scognet *mapp = NULL; 617166063Scognet return (error); 618166063Scognet } 619166063Scognet } 620166063Scognet bz = dmat->bounce_zone; 621166063Scognet 622166063Scognet /* Initialize the new map */ 623166063Scognet STAILQ_INIT(&((*mapp)->bpages)); 624166063Scognet 625166063Scognet /* 626166063Scognet * Attempt to add pages to our pool on a per-instance 627166063Scognet * basis up to a sane limit. 628166063Scognet */ 629166063Scognet maxpages = MAX_BPAGES; 630166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 631188403Scognet || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 632166063Scognet int pages; 633166063Scognet 634166063Scognet pages = MAX(atop(dmat->maxsize), 1); 635166063Scognet pages = MIN(maxpages - bz->total_bpages, pages); 636166063Scognet pages = MAX(pages, 1); 637166063Scognet if (alloc_bounce_pages(dmat, pages) < pages) 638166063Scognet error = ENOMEM; 639166063Scognet 640166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 641166063Scognet if (error == 0) 642166063Scognet dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 643166063Scognet } else { 644166063Scognet error = 0; 645166063Scognet } 646166063Scognet } 647188403Scognet bz->map_count++; 648166063Scognet } 649246713Skib map->sync_count = 0; 650246713Skib map->slist = slist; 651143294Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 652143284Smux __func__, dmat, dmat->flags, error); 653140313Scognet 654129198Scognet return (0); 655129198Scognet} 656129198Scognet 657129198Scognet/* 658129198Scognet * Destroy a handle for mapping from kva/uva/physical 659129198Scognet * address space into bus device space. 660129198Scognet */ 661129198Scognetint 662129198Scognetbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 663129198Scognet{ 664135644Scognet 665246713Skib if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 666166063Scognet CTR3(KTR_BUSDMA, "%s: tag %p error %d", 667166063Scognet __func__, dmat, EBUSY); 668166063Scognet return (EBUSY); 669166063Scognet } 670246713Skib free(map->slist, M_DEVBUF); 671244471Scognet uma_zfree(dmamap_zone, map); 672188403Scognet if (dmat->bounce_zone) 673188403Scognet dmat->bounce_zone->map_count--; 674143294Smux CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 675129198Scognet return (0); 676129198Scognet} 677129198Scognet 678129198Scognet/* 679244471Scognet * Allocate a piece of memory that can be efficiently mapped into bus device 680244471Scognet * space based on the constraints listed in the dma tag. Returns a pointer to 681244471Scognet * the allocated memory, and a pointer to an associated bus_dmamap. 682129198Scognet */ 683129198Scognetint 684244471Scognetbus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 685129198Scognet bus_dmamap_t *mapp) 686129198Scognet{ 687246713Skib struct sync_list *slist; 688244471Scognet void * vaddr; 689244471Scognet struct busdma_bufzone *bufzone; 690244471Scognet busdma_bufalloc_t ba; 691244471Scognet bus_dmamap_t map; 692129198Scognet int mflags; 693244471Scognet vm_memattr_t memattr; 694129198Scognet 695129198Scognet if (flags & BUS_DMA_NOWAIT) 696129198Scognet mflags = M_NOWAIT; 697129198Scognet else 698129198Scognet mflags = M_WAITOK; 699244471Scognet /* 700244471Scognet * If the tag's segments haven't been allocated yet we need to do it 701244471Scognet * now, because we can't sleep for resources at map load time. 702244471Scognet */ 703244471Scognet if (dmat->segments == NULL) 704244471Scognet dmat->segments = malloc(dmat->nsegments * 705244471Scognet sizeof(*dmat->segments), M_DEVBUF, mflags); 706244471Scognet 707246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 708246713Skib if (slist == NULL) 709246713Skib return (ENOMEM); 710244471Scognet map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 711246713Skib if (map == NULL) { 712246713Skib free(slist, M_DEVBUF); 713244471Scognet return (ENOMEM); 714246713Skib } 715244471Scognet if (flags & BUS_DMA_COHERENT) { 716244471Scognet memattr = VM_MEMATTR_UNCACHEABLE; 717244471Scognet ba = coherent_allocator; 718244471Scognet map->flags |= DMAMAP_COHERENT; 719244471Scognet } else { 720244471Scognet memattr = VM_MEMATTR_DEFAULT; 721244471Scognet ba = standard_allocator; 722240177Sjhb } 723244471Scognet /* All buffers we allocate are cache-aligned. */ 724244471Scognet map->flags |= DMAMAP_CACHE_ALIGNED; 725244471Scognet 726129198Scognet if (flags & BUS_DMA_ZERO) 727129198Scognet mflags |= M_ZERO; 728129198Scognet 729244471Scognet /* 730244471Scognet * Try to find a bufzone in the allocator that holds a cache of buffers 731244471Scognet * of the right size for this request. If the buffer is too big to be 732244471Scognet * held in the allocator cache, this returns NULL. 733244471Scognet */ 734244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 735244471Scognet 736244471Scognet /* 737244471Scognet * Allocate the buffer from the uma(9) allocator if... 738244471Scognet * - It's small enough to be in the allocator (bufzone not NULL). 739244471Scognet * - The alignment constraint isn't larger than the allocation size 740244471Scognet * (the allocator aligns buffers to their size boundaries). 741244471Scognet * - There's no need to handle lowaddr/highaddr exclusion zones. 742244471Scognet * else allocate non-contiguous pages if... 743244471Scognet * - The page count that could get allocated doesn't exceed nsegments. 744244471Scognet * - The alignment constraint isn't larger than a page boundary. 745244471Scognet * - There are no boundary-crossing constraints. 746244471Scognet * else allocate a block of contiguous pages because one or more of the 747244471Scognet * constraints is something that only the contig allocator can fulfill. 748244471Scognet */ 749244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 750244471Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 751244471Scognet vaddr = uma_zalloc(bufzone->umazone, mflags); 752244471Scognet } else if (dmat->nsegments >= btoc(dmat->maxsize) && 753244471Scognet dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 754254025Sjeff vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 755244471Scognet mflags, 0, dmat->lowaddr, memattr); 756244471Scognet } else { 757254025Sjeff vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 758244471Scognet mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 759244471Scognet memattr); 760135644Scognet } 761244471Scognet if (vaddr == NULL) { 762246713Skib free(slist, M_DEVBUF); 763244471Scognet uma_zfree(dmamap_zone, map); 764244471Scognet map = NULL; 765246713Skib } else { 766246713Skib map->slist = slist; 767246713Skib map->sync_count = 0; 768129198Scognet } 769244471Scognet *vaddrp = vaddr; 770244471Scognet *mapp = map; 771244471Scognet 772244471Scognet return (vaddr == NULL ? ENOMEM : 0); 773129198Scognet} 774129198Scognet 775129198Scognet/* 776244471Scognet * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 777244471Scognet * its associated map. 778129198Scognet */ 779129198Scognetvoid 780129198Scognetbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 781129198Scognet{ 782244471Scognet struct busdma_bufzone *bufzone; 783244471Scognet busdma_bufalloc_t ba; 784244471Scognet 785244471Scognet if (map->flags & DMAMAP_COHERENT) 786244471Scognet ba = coherent_allocator; 787246713Skib else 788244471Scognet ba = standard_allocator; 789246713Skib uma_zfree(dmamap_zone, map); 790244471Scognet 791246713Skib free(map->slist, M_DEVBUF); 792244471Scognet /* Be careful not to access map from here on. */ 793244471Scognet 794244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 795244471Scognet 796244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 797166063Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 798244471Scognet uma_zfree(bufzone->umazone, vaddr); 799244471Scognet else 800254025Sjeff kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 801129198Scognet} 802129198Scognet 803246713Skibstatic void 804246713Skib_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 805246713Skib bus_size_t buflen, int flags) 806246713Skib{ 807246713Skib bus_addr_t curaddr; 808246713Skib bus_size_t sgsize; 809246713Skib 810257203Sian if (map->pagesneeded == 0) { 811246713Skib CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 812246713Skib dmat->lowaddr, dmat->boundary, dmat->alignment); 813246713Skib CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 814246713Skib map, map->pagesneeded); 815246713Skib /* 816246713Skib * Count the number of bounce pages 817246713Skib * needed in order to complete this transfer 818246713Skib */ 819246713Skib curaddr = buf; 820246713Skib while (buflen != 0) { 821246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 822246713Skib if (run_filter(dmat, curaddr) != 0) { 823246713Skib sgsize = MIN(sgsize, PAGE_SIZE); 824246713Skib map->pagesneeded++; 825246713Skib } 826246713Skib curaddr += sgsize; 827246713Skib buflen -= sgsize; 828246713Skib } 829246713Skib CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 830246713Skib } 831246713Skib} 832246713Skib 833246713Skibstatic void 834191011Skib_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 835191011Skib void *buf, bus_size_t buflen, int flags) 836166063Scognet{ 837166063Scognet vm_offset_t vaddr; 838166063Scognet vm_offset_t vendaddr; 839166063Scognet bus_addr_t paddr; 840166063Scognet 841257203Sian if (map->pagesneeded == 0) { 842185494Sstas CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 843185494Sstas dmat->lowaddr, dmat->boundary, dmat->alignment); 844170406Scognet CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 845170406Scognet map, map->pagesneeded); 846166063Scognet /* 847166063Scognet * Count the number of bounce pages 848166063Scognet * needed in order to complete this transfer 849166063Scognet */ 850166063Scognet vaddr = trunc_page((vm_offset_t)buf); 851166063Scognet vendaddr = (vm_offset_t)buf + buflen; 852166063Scognet 853166063Scognet while (vaddr < vendaddr) { 854246713Skib if (__predict_true(pmap == kernel_pmap)) 855191438Sjhb paddr = pmap_kextract(vaddr); 856191438Sjhb else 857191011Skib paddr = pmap_extract(pmap, vaddr); 858246713Skib if (run_filter(dmat, paddr) != 0) 859166063Scognet map->pagesneeded++; 860166063Scognet vaddr += PAGE_SIZE; 861166063Scognet } 862166063Scognet CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 863166063Scognet } 864246713Skib} 865166063Scognet 866246713Skibstatic int 867246713Skib_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 868246713Skib{ 869246713Skib 870166063Scognet /* Reserve Necessary Bounce Pages */ 871246713Skib mtx_lock(&bounce_lock); 872246713Skib if (flags & BUS_DMA_NOWAIT) { 873246713Skib if (reserve_bounce_pages(dmat, map, 0) != 0) { 874246713Skib mtx_unlock(&bounce_lock); 875246713Skib return (ENOMEM); 876166063Scognet } 877246713Skib } else { 878246713Skib if (reserve_bounce_pages(dmat, map, 1) != 0) { 879246713Skib /* Queue us for resources */ 880246713Skib STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 881246713Skib mtx_unlock(&bounce_lock); 882246713Skib return (EINPROGRESS); 883246713Skib } 884166063Scognet } 885246713Skib mtx_unlock(&bounce_lock); 886166063Scognet 887166063Scognet return (0); 888166063Scognet} 889166063Scognet 890129198Scognet/* 891246713Skib * Add a single contiguous physical range to the segment list. 892246713Skib */ 893246713Skibstatic int 894246713Skib_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 895246713Skib bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 896246713Skib{ 897246713Skib bus_addr_t baddr, bmask; 898246713Skib int seg; 899246713Skib 900246713Skib /* 901246713Skib * Make sure we don't cross any boundaries. 902246713Skib */ 903246713Skib bmask = ~(dmat->boundary - 1); 904246713Skib if (dmat->boundary > 0) { 905246713Skib baddr = (curaddr + dmat->boundary) & bmask; 906246713Skib if (sgsize > (baddr - curaddr)) 907246713Skib sgsize = (baddr - curaddr); 908246713Skib } 909246713Skib if (dmat->ranges) { 910246713Skib struct arm32_dma_range *dr; 911246713Skib 912246713Skib dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 913246713Skib curaddr); 914246713Skib if (dr == NULL) 915246881Sian return (0); 916246713Skib /* 917246713Skib * In a valid DMA range. Translate the physical 918246713Skib * memory address to an address in the DMA window. 919246713Skib */ 920246713Skib curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 921246713Skib 922246713Skib } 923246713Skib 924246713Skib seg = *segp; 925246713Skib /* 926246713Skib * Insert chunk into a segment, coalescing with 927246713Skib * the previous segment if possible. 928246713Skib */ 929246713Skib if (seg >= 0 && 930246713Skib curaddr == segs[seg].ds_addr + segs[seg].ds_len && 931246713Skib (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 932246713Skib (dmat->boundary == 0 || 933246713Skib (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 934246713Skib segs[seg].ds_len += sgsize; 935246713Skib } else { 936246713Skib if (++seg >= dmat->nsegments) 937246881Sian return (0); 938246713Skib segs[seg].ds_addr = curaddr; 939246713Skib segs[seg].ds_len = sgsize; 940246713Skib } 941246713Skib *segp = seg; 942246881Sian return (sgsize); 943246713Skib} 944246713Skib 945246713Skib/* 946246713Skib * Utility function to load a physical buffer. segp contains 947246713Skib * the starting segment on entrace, and the ending segment on exit. 948246713Skib */ 949246713Skibint 950246713Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 951246713Skib bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 952246713Skib{ 953246713Skib bus_size_t sgsize; 954246713Skib bus_addr_t curaddr; 955246713Skib int error; 956246713Skib 957246713Skib if (segs == NULL) 958246713Skib segs = dmat->segments; 959246713Skib 960246713Skib if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 961246713Skib _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 962246713Skib if (map->pagesneeded != 0) { 963246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 964246713Skib if (error) 965246713Skib return (error); 966246713Skib } 967246713Skib } 968246713Skib 969246713Skib while (buflen > 0) { 970246713Skib curaddr = buf; 971246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 972246713Skib if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 973246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 974246713Skib sgsize = MIN(sgsize, PAGE_SIZE); 975246713Skib curaddr = add_bounce_page(dmat, map, 0, curaddr, 976246713Skib sgsize); 977246713Skib } 978246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 979246713Skib segp); 980246713Skib if (sgsize == 0) 981246713Skib break; 982246713Skib buf += sgsize; 983246713Skib buflen -= sgsize; 984246713Skib } 985246713Skib 986246713Skib /* 987246713Skib * Did we fit? 988246713Skib */ 989246713Skib if (buflen != 0) { 990246713Skib _bus_dmamap_unload(dmat, map); 991246713Skib return (EFBIG); /* XXX better return value here? */ 992246713Skib } 993246713Skib return (0); 994246713Skib} 995257228Skib 996257228Skibint 997257228Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 998257228Skib struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 999257228Skib bus_dma_segment_t *segs, int *segp) 1000257228Skib{ 1001257228Skib 1002257228Skib return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1003257228Skib segs, segp)); 1004257228Skib} 1005257228Skib 1006246713Skib/* 1007246713Skib * Utility function to load a linear buffer. segp contains 1008129198Scognet * the starting segment on entrance, and the ending segment on exit. 1009129198Scognet */ 1010246713Skibint 1011246713Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1012246713Skib bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1013246713Skib int *segp) 1014129198Scognet{ 1015129198Scognet bus_size_t sgsize; 1016246713Skib bus_addr_t curaddr; 1017246713Skib struct sync_list *sl; 1018129198Scognet vm_offset_t vaddr = (vm_offset_t)buf; 1019129198Scognet int error = 0; 1020129198Scognet 1021246713Skib if (segs == NULL) 1022246713Skib segs = dmat->segments; 1023246713Skib if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1024246713Skib map->flags |= DMAMAP_CACHE_ALIGNED; 1025129198Scognet 1026166063Scognet if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1027246713Skib _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1028246713Skib if (map->pagesneeded != 0) { 1029246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 1030246713Skib if (error) 1031246713Skib return (error); 1032246713Skib } 1033166063Scognet } 1034140313Scognet CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1035140313Scognet "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1036140313Scognet 1037246713Skib while (buflen > 0) { 1038129198Scognet /* 1039129198Scognet * Get the physical address for this segment. 1040129198Scognet */ 1041246713Skib if (__predict_true(pmap == kernel_pmap)) { 1042246158Skib curaddr = pmap_kextract(vaddr); 1043129198Scognet } else { 1044129198Scognet curaddr = pmap_extract(pmap, vaddr); 1045135644Scognet map->flags &= ~DMAMAP_COHERENT; 1046129198Scognet } 1047129198Scognet 1048129198Scognet /* 1049129198Scognet * Compute the segment size, and adjust counts. 1050129198Scognet */ 1051129198Scognet sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 1052170086Syongari if (sgsize > dmat->maxsegsz) 1053170086Syongari sgsize = dmat->maxsegsz; 1054129198Scognet if (buflen < sgsize) 1055129198Scognet sgsize = buflen; 1056129198Scognet 1057166063Scognet if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1058246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1059246713Skib curaddr = add_bounce_page(dmat, map, vaddr, curaddr, 1060246713Skib sgsize); 1061137760Scognet } else { 1062246713Skib sl = &map->slist[map->sync_count - 1]; 1063246713Skib if (map->sync_count == 0 || 1064246713Skib vaddr != sl->vaddr + sl->datacount) { 1065246713Skib if (++map->sync_count > dmat->nsegments) 1066246713Skib goto cleanup; 1067246713Skib sl++; 1068246713Skib sl->vaddr = vaddr; 1069246713Skib sl->datacount = sgsize; 1070246713Skib sl->busaddr = curaddr; 1071246713Skib } else 1072246713Skib sl->datacount += sgsize; 1073129198Scognet } 1074246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1075246713Skib segp); 1076246713Skib if (sgsize == 0) 1077135644Scognet break; 1078129198Scognet vaddr += sgsize; 1079129198Scognet buflen -= sgsize; 1080129198Scognet } 1081129198Scognet 1082246713Skibcleanup: 1083129198Scognet /* 1084129198Scognet * Did we fit? 1085129198Scognet */ 1086246713Skib if (buflen != 0) { 1087246713Skib _bus_dmamap_unload(dmat, map); 1088246713Skib return (EFBIG); /* XXX better return value here? */ 1089246713Skib } 1090246713Skib return (0); 1091129198Scognet} 1092129198Scognet 1093246713Skibvoid 1094246713Skib__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 1095246713Skib struct memdesc *mem, bus_dmamap_callback_t *callback, 1096246713Skib void *callback_arg) 1097140682Scognet{ 1098140682Scognet 1099143671Sjmg KASSERT(dmat != NULL, ("dmatag is NULL")); 1100143671Sjmg KASSERT(map != NULL, ("dmamap is NULL")); 1101246713Skib map->mem = *mem; 1102166063Scognet map->callback = callback; 1103166063Scognet map->callback_arg = callback_arg; 1104140682Scognet} 1105140682Scognet 1106246713Skibbus_dma_segment_t * 1107246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1108246713Skib bus_dma_segment_t *segs, int nsegs, int error) 1109129198Scognet{ 1110129198Scognet 1111246713Skib if (segs == NULL) 1112246713Skib segs = dmat->segments; 1113246713Skib return (segs); 1114129198Scognet} 1115129198Scognet 1116129198Scognet/* 1117135644Scognet * Release the mapping held by map. 1118129198Scognet */ 1119129198Scognetvoid 1120143655Sjmg_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1121150893Scognet{ 1122166063Scognet struct bounce_page *bpage; 1123166063Scognet 1124166063Scognet while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1125166063Scognet STAILQ_REMOVE_HEAD(&map->bpages, links); 1126166063Scognet free_bounce_page(dmat, bpage); 1127166063Scognet } 1128246713Skib map->sync_count = 0; 1129129198Scognet return; 1130129198Scognet} 1131129198Scognet 1132169761Scognetstatic void 1133246713Skibbus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1134246713Skib int bufaligned) 1135135644Scognet{ 1136166063Scognet char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1137234561Smarius register_t s; 1138236991Simp int partial; 1139135644Scognet 1140171890Scognet if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1141246713Skib cpu_dcache_wb_range(buf, len); 1142246713Skib cpu_l2cache_wb_range(buf, len); 1143171623Scognet } 1144244471Scognet 1145244471Scognet /* 1146244471Scognet * If the caller promises the buffer is properly aligned to a cache line 1147244471Scognet * (even if the call parms make it look like it isn't) we can avoid 1148244471Scognet * attempting to preserve the non-DMA part of the cache line in the 1149244471Scognet * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1150244471Scognet * 1151244471Scognet * This covers the case of mbufs, where we know how they're aligned and 1152244471Scognet * know the CPU doesn't touch the header in front of the DMA data area 1153244471Scognet * during the IO, but it may have touched it right before invoking the 1154244471Scognet * sync, so a PREREAD writeback is required. 1155244471Scognet * 1156244471Scognet * It also handles buffers we created in bus_dmamem_alloc(), which are 1157244471Scognet * always aligned and padded to cache line size even if the IO length 1158244471Scognet * isn't a multiple of cache line size. In this case the PREREAD 1159244471Scognet * writeback probably isn't required, but it's harmless. 1160244471Scognet */ 1161234561Smarius partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1162244471Scognet 1163171623Scognet if (op & BUS_DMASYNC_PREREAD) { 1164234561Smarius if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1165246713Skib cpu_dcache_inv_range(buf, len); 1166246713Skib cpu_l2cache_inv_range(buf, len); 1167171890Scognet } else { 1168246713Skib cpu_dcache_wbinv_range(buf, len); 1169246713Skib cpu_l2cache_wbinv_range(buf, len); 1170171890Scognet } 1171171623Scognet } 1172166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1173244471Scognet if (partial && !bufaligned) { 1174234561Smarius s = intr_disable(); 1175246713Skib if (buf & arm_dcache_align_mask) 1176246713Skib memcpy(_tmp_cl, (void *)(buf & 1177234561Smarius ~arm_dcache_align_mask), 1178246713Skib buf & arm_dcache_align_mask); 1179246713Skib if ((buf + len) & arm_dcache_align_mask) 1180236991Simp memcpy(_tmp_clend, 1181246713Skib (void *)(buf + len), 1182246713Skib arm_dcache_align - 1183246713Skib ((buf + len) & arm_dcache_align_mask)); 1184171623Scognet } 1185246713Skib cpu_dcache_inv_range(buf, len); 1186246713Skib cpu_l2cache_inv_range(buf, len); 1187244471Scognet if (partial && !bufaligned) { 1188246713Skib if (buf & arm_dcache_align_mask) 1189246713Skib memcpy((void *)(buf & 1190236991Simp ~arm_dcache_align_mask), _tmp_cl, 1191246713Skib buf & arm_dcache_align_mask); 1192246713Skib if ((buf + len) & arm_dcache_align_mask) 1193246713Skib memcpy((void *)(buf + len), 1194236991Simp _tmp_clend, arm_dcache_align - 1195246713Skib ((buf + len) & arm_dcache_align_mask)); 1196234561Smarius intr_restore(s); 1197234561Smarius } 1198135644Scognet } 1199135644Scognet} 1200135644Scognet 1201166063Scognetstatic void 1202166063Scognet_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1203166063Scognet{ 1204166063Scognet struct bounce_page *bpage; 1205166063Scognet 1206166063Scognet STAILQ_FOREACH(bpage, &map->bpages, links) { 1207166063Scognet if (op & BUS_DMASYNC_PREWRITE) { 1208246713Skib if (bpage->datavaddr != 0) 1209257201Sian bcopy((void *)bpage->datavaddr, 1210257201Sian (void *)bpage->vaddr, bpage->datacount); 1211246713Skib else 1212246713Skib physcopyout(bpage->dataaddr, 1213257201Sian (void *)bpage->vaddr,bpage->datacount); 1214257201Sian cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1215257201Sian cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1216187911Sthompsa dmat->bounce_zone->total_bounced++; 1217166063Scognet } 1218166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1219257201Sian cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1220257201Sian cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1221246713Skib if (bpage->datavaddr != 0) 1222257201Sian bcopy((void *)bpage->vaddr, 1223246713Skib (void *)bpage->datavaddr, bpage->datacount); 1224246713Skib else 1225257201Sian physcopyin((void *)bpage->vaddr, 1226246713Skib bpage->dataaddr, bpage->datacount); 1227187911Sthompsa dmat->bounce_zone->total_bounced++; 1228166063Scognet } 1229166063Scognet } 1230166063Scognet} 1231166063Scognet 1232129198Scognetvoid 1233143655Sjmg_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1234129198Scognet{ 1235246713Skib struct sync_list *sl, *end; 1236244471Scognet int bufaligned; 1237244471Scognet 1238159107Scognet if (op == BUS_DMASYNC_POSTWRITE) 1239129198Scognet return; 1240244471Scognet if (map->flags & DMAMAP_COHERENT) 1241244471Scognet goto drain; 1242166063Scognet if (STAILQ_FIRST(&map->bpages)) 1243166063Scognet _bus_dmamap_sync_bp(dmat, map, op); 1244143294Smux CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1245244471Scognet bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1246246713Skib if (map->sync_count) { 1247246713Skib end = &map->slist[map->sync_count]; 1248246713Skib for (sl = &map->slist[0]; sl != end; sl++) 1249246713Skib bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, 1250244471Scognet bufaligned); 1251129198Scognet } 1252244471Scognet 1253244471Scognetdrain: 1254244471Scognet 1255135644Scognet cpu_drain_writebuf(); 1256129198Scognet} 1257166063Scognet 1258166063Scognetstatic void 1259166063Scognetinit_bounce_pages(void *dummy __unused) 1260166063Scognet{ 1261166063Scognet 1262166063Scognet total_bpages = 0; 1263166063Scognet STAILQ_INIT(&bounce_zone_list); 1264166063Scognet STAILQ_INIT(&bounce_map_waitinglist); 1265166063Scognet STAILQ_INIT(&bounce_map_callbacklist); 1266166063Scognet mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1267166063Scognet} 1268166063ScognetSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1269166063Scognet 1270166063Scognetstatic struct sysctl_ctx_list * 1271166063Scognetbusdma_sysctl_tree(struct bounce_zone *bz) 1272166063Scognet{ 1273166063Scognet return (&bz->sysctl_tree); 1274166063Scognet} 1275166063Scognet 1276166063Scognetstatic struct sysctl_oid * 1277166063Scognetbusdma_sysctl_tree_top(struct bounce_zone *bz) 1278166063Scognet{ 1279166063Scognet return (bz->sysctl_tree_top); 1280166063Scognet} 1281166063Scognet 1282166063Scognetstatic int 1283166063Scognetalloc_bounce_zone(bus_dma_tag_t dmat) 1284166063Scognet{ 1285166063Scognet struct bounce_zone *bz; 1286166063Scognet 1287166063Scognet /* Check to see if we already have a suitable zone */ 1288166063Scognet STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1289166063Scognet if ((dmat->alignment <= bz->alignment) 1290166063Scognet && (dmat->lowaddr >= bz->lowaddr)) { 1291166063Scognet dmat->bounce_zone = bz; 1292166063Scognet return (0); 1293166063Scognet } 1294166063Scognet } 1295166063Scognet 1296166063Scognet if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1297166063Scognet M_NOWAIT | M_ZERO)) == NULL) 1298166063Scognet return (ENOMEM); 1299166063Scognet 1300166063Scognet STAILQ_INIT(&bz->bounce_page_list); 1301166063Scognet bz->free_bpages = 0; 1302166063Scognet bz->reserved_bpages = 0; 1303166063Scognet bz->active_bpages = 0; 1304166063Scognet bz->lowaddr = dmat->lowaddr; 1305191438Sjhb bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1306188403Scognet bz->map_count = 0; 1307166063Scognet snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1308166063Scognet busdma_zonecount++; 1309166063Scognet snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1310166063Scognet STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1311166063Scognet dmat->bounce_zone = bz; 1312166063Scognet 1313166063Scognet sysctl_ctx_init(&bz->sysctl_tree); 1314166063Scognet bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1315166063Scognet SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1316166063Scognet CTLFLAG_RD, 0, ""); 1317166063Scognet if (bz->sysctl_tree_top == NULL) { 1318166063Scognet sysctl_ctx_free(&bz->sysctl_tree); 1319166063Scognet return (0); /* XXX error code? */ 1320166063Scognet } 1321166063Scognet 1322166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1323166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1324166063Scognet "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1325166063Scognet "Total bounce pages"); 1326166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1327166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1328166063Scognet "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1329166063Scognet "Free bounce pages"); 1330166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1331166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1332166063Scognet "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1333166063Scognet "Reserved bounce pages"); 1334166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1335166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1336166063Scognet "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1337166063Scognet "Active bounce pages"); 1338166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1339166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1340166063Scognet "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1341166063Scognet "Total bounce requests"); 1342166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1343166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1344166063Scognet "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1345166063Scognet "Total bounce requests that were deferred"); 1346166063Scognet SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1347166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1348166063Scognet "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1349273377Shselasky SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1350166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1351273377Shselasky "alignment", CTLFLAG_RD, &bz->alignment, ""); 1352166063Scognet 1353166063Scognet return (0); 1354166063Scognet} 1355166063Scognet 1356166063Scognetstatic int 1357166063Scognetalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1358166063Scognet{ 1359166063Scognet struct bounce_zone *bz; 1360166063Scognet int count; 1361166063Scognet 1362166063Scognet bz = dmat->bounce_zone; 1363166063Scognet count = 0; 1364166063Scognet while (numpages > 0) { 1365166063Scognet struct bounce_page *bpage; 1366166063Scognet 1367166063Scognet bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1368166063Scognet M_NOWAIT | M_ZERO); 1369166063Scognet 1370166063Scognet if (bpage == NULL) 1371166063Scognet break; 1372166063Scognet bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1373166063Scognet M_NOWAIT, 0ul, 1374166063Scognet bz->lowaddr, 1375166063Scognet PAGE_SIZE, 1376191438Sjhb 0); 1377166063Scognet if (bpage->vaddr == 0) { 1378166063Scognet free(bpage, M_DEVBUF); 1379166063Scognet break; 1380166063Scognet } 1381166063Scognet bpage->busaddr = pmap_kextract(bpage->vaddr); 1382166063Scognet mtx_lock(&bounce_lock); 1383166063Scognet STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1384166063Scognet total_bpages++; 1385166063Scognet bz->total_bpages++; 1386166063Scognet bz->free_bpages++; 1387166063Scognet mtx_unlock(&bounce_lock); 1388166063Scognet count++; 1389166063Scognet numpages--; 1390166063Scognet } 1391166063Scognet return (count); 1392166063Scognet} 1393166063Scognet 1394166063Scognetstatic int 1395166063Scognetreserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1396166063Scognet{ 1397166063Scognet struct bounce_zone *bz; 1398166063Scognet int pages; 1399166063Scognet 1400166063Scognet mtx_assert(&bounce_lock, MA_OWNED); 1401166063Scognet bz = dmat->bounce_zone; 1402166063Scognet pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1403166063Scognet if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1404166063Scognet return (map->pagesneeded - (map->pagesreserved + pages)); 1405166063Scognet bz->free_bpages -= pages; 1406166063Scognet bz->reserved_bpages += pages; 1407166063Scognet map->pagesreserved += pages; 1408166063Scognet pages = map->pagesneeded - map->pagesreserved; 1409166063Scognet 1410166063Scognet return (pages); 1411166063Scognet} 1412166063Scognet 1413166063Scognetstatic bus_addr_t 1414166063Scognetadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1415246713Skib bus_addr_t addr, bus_size_t size) 1416166063Scognet{ 1417166063Scognet struct bounce_zone *bz; 1418166063Scognet struct bounce_page *bpage; 1419166063Scognet 1420166063Scognet KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1421170406Scognet KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1422166063Scognet 1423166063Scognet bz = dmat->bounce_zone; 1424166063Scognet if (map->pagesneeded == 0) 1425166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1426166063Scognet map->pagesneeded--; 1427166063Scognet 1428166063Scognet if (map->pagesreserved == 0) 1429166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1430166063Scognet map->pagesreserved--; 1431166063Scognet 1432166063Scognet mtx_lock(&bounce_lock); 1433166063Scognet bpage = STAILQ_FIRST(&bz->bounce_page_list); 1434166063Scognet if (bpage == NULL) 1435166063Scognet panic("add_bounce_page: free page list is empty"); 1436166063Scognet 1437166063Scognet STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1438166063Scognet bz->reserved_bpages--; 1439166063Scognet bz->active_bpages++; 1440166063Scognet mtx_unlock(&bounce_lock); 1441166063Scognet 1442188350Simp if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1443191201Sjhb /* Page offset needs to be preserved. */ 1444188350Simp bpage->vaddr |= vaddr & PAGE_MASK; 1445188350Simp bpage->busaddr |= vaddr & PAGE_MASK; 1446188350Simp } 1447166063Scognet bpage->datavaddr = vaddr; 1448246713Skib bpage->dataaddr = addr; 1449166063Scognet bpage->datacount = size; 1450166063Scognet STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1451166063Scognet return (bpage->busaddr); 1452166063Scognet} 1453166063Scognet 1454166063Scognetstatic void 1455166063Scognetfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1456166063Scognet{ 1457166063Scognet struct bus_dmamap *map; 1458166063Scognet struct bounce_zone *bz; 1459166063Scognet 1460166063Scognet bz = dmat->bounce_zone; 1461166063Scognet bpage->datavaddr = 0; 1462166063Scognet bpage->datacount = 0; 1463191201Sjhb if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1464191201Sjhb /* 1465191201Sjhb * Reset the bounce page to start at offset 0. Other uses 1466191201Sjhb * of this bounce page may need to store a full page of 1467191201Sjhb * data and/or assume it starts on a page boundary. 1468191201Sjhb */ 1469191201Sjhb bpage->vaddr &= ~PAGE_MASK; 1470191201Sjhb bpage->busaddr &= ~PAGE_MASK; 1471191201Sjhb } 1472166063Scognet 1473166063Scognet mtx_lock(&bounce_lock); 1474166063Scognet STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1475166063Scognet bz->free_bpages++; 1476166063Scognet bz->active_bpages--; 1477166063Scognet if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1478166063Scognet if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1479166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1480166063Scognet STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1481166063Scognet map, links); 1482166063Scognet busdma_swi_pending = 1; 1483166063Scognet bz->total_deferred++; 1484166063Scognet swi_sched(vm_ih, 0); 1485166063Scognet } 1486166063Scognet } 1487166063Scognet mtx_unlock(&bounce_lock); 1488166063Scognet} 1489166063Scognet 1490166063Scognetvoid 1491166063Scognetbusdma_swi(void) 1492166063Scognet{ 1493166063Scognet bus_dma_tag_t dmat; 1494166063Scognet struct bus_dmamap *map; 1495166063Scognet 1496166063Scognet mtx_lock(&bounce_lock); 1497166063Scognet while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1498166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1499166063Scognet mtx_unlock(&bounce_lock); 1500166063Scognet dmat = map->dmat; 1501166063Scognet (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1502246713Skib bus_dmamap_load_mem(map->dmat, map, &map->mem, 1503246713Skib map->callback, map->callback_arg, BUS_DMA_WAITOK); 1504166063Scognet (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1505166063Scognet mtx_lock(&bounce_lock); 1506166063Scognet } 1507166063Scognet mtx_unlock(&bounce_lock); 1508166063Scognet} 1509