1139735Simp/*- 2244471Scognet * Copyright (c) 2012 Ian Lepore 3129198Scognet * Copyright (c) 2004 Olivier Houchard 4129198Scognet * Copyright (c) 2002 Peter Grehan 5129198Scognet * Copyright (c) 1997, 1998 Justin T. Gibbs. 6129198Scognet * All rights reserved. 7129198Scognet * 8129198Scognet * Redistribution and use in source and binary forms, with or without 9129198Scognet * modification, are permitted provided that the following conditions 10129198Scognet * are met: 11129198Scognet * 1. Redistributions of source code must retain the above copyright 12129198Scognet * notice, this list of conditions, and the following disclaimer, 13129198Scognet * without modification, immediately at the beginning of the file. 14129198Scognet * 2. The name of the author may not be used to endorse or promote products 15129198Scognet * derived from this software without specific prior written permission. 16129198Scognet * 17129198Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18129198Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19129198Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20129198Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21129198Scognet * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22129198Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23129198Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24129198Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25129198Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26129198Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27129198Scognet * SUCH DAMAGE. 28129198Scognet * 29129198Scognet * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30129198Scognet */ 31129198Scognet 32129198Scognet#include <sys/cdefs.h> 33129198Scognet__FBSDID("$FreeBSD: stable/10/sys/arm/arm/busdma_machdep.c 318977 2017-05-27 08:17:59Z hselasky $"); 34129198Scognet 35129198Scognet/* 36244471Scognet * ARM bus dma support routines. 37244471Scognet * 38244471Scognet * XXX Things to investigate / fix some day... 39244471Scognet * - What is the earliest that this API can be called? Could there be any 40244471Scognet * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41244471Scognet * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42244471Scognet * bus_dmamap_load() function. This code has historically (and still does) 43244471Scognet * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44244471Scognet * error checking because some resource management calls would become WAITOK 45244471Scognet * and thus "cannot fail." 46244471Scognet * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47244471Scognet * creation time, and the result stored in the tag. 48244471Scognet * - It should be possible to take some shortcuts when mapping a buffer we know 49244471Scognet * came from the uma(9) allocators based on what we know about such buffers 50244471Scognet * (aligned, contiguous, etc). 51244471Scognet * - The allocation of bounce pages could probably be cleaned up, then we could 52244471Scognet * retire arm_remap_nocache(). 53129198Scognet */ 54129198Scognet 55129198Scognet#define _ARM32_BUS_DMA_PRIVATE 56129198Scognet#include <sys/param.h> 57129198Scognet#include <sys/systm.h> 58129198Scognet#include <sys/malloc.h> 59129198Scognet#include <sys/bus.h> 60244471Scognet#include <sys/busdma_bufalloc.h> 61129198Scognet#include <sys/interrupt.h> 62129198Scognet#include <sys/lock.h> 63129198Scognet#include <sys/proc.h> 64246713Skib#include <sys/memdesc.h> 65129198Scognet#include <sys/mutex.h> 66140310Scognet#include <sys/ktr.h> 67146597Scognet#include <sys/kernel.h> 68166063Scognet#include <sys/sysctl.h> 69246713Skib#include <sys/uio.h> 70129198Scognet 71244471Scognet#include <vm/uma.h> 72129198Scognet#include <vm/vm.h> 73244471Scognet#include <vm/vm_extern.h> 74244471Scognet#include <vm/vm_kern.h> 75129198Scognet#include <vm/vm_page.h> 76129198Scognet#include <vm/vm_map.h> 77129198Scognet 78129198Scognet#include <machine/atomic.h> 79129198Scognet#include <machine/bus.h> 80129198Scognet#include <machine/cpufunc.h> 81166063Scognet#include <machine/md_var.h> 82129198Scognet 83166063Scognet#define MAX_BPAGES 64 84166063Scognet#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85166063Scognet#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86166063Scognet 87166063Scognetstruct bounce_zone; 88166063Scognet 89129198Scognetstruct bus_dma_tag { 90129198Scognet bus_dma_tag_t parent; 91129198Scognet bus_size_t alignment; 92232356Sjhb bus_addr_t boundary; 93129198Scognet bus_addr_t lowaddr; 94129198Scognet bus_addr_t highaddr; 95129198Scognet bus_dma_filter_t *filter; 96129198Scognet void *filterarg; 97129198Scognet bus_size_t maxsize; 98129198Scognet u_int nsegments; 99129198Scognet bus_size_t maxsegsz; 100129198Scognet int flags; 101129198Scognet int ref_count; 102129198Scognet int map_count; 103129198Scognet bus_dma_lock_t *lockfunc; 104129198Scognet void *lockfuncarg; 105129198Scognet /* 106129198Scognet * DMA range for this tag. If the page doesn't fall within 107129198Scognet * one of these ranges, an error is returned. The caller 108129198Scognet * may then decide what to do with the transfer. If the 109129198Scognet * range pointer is NULL, it is ignored. 110129198Scognet */ 111129198Scognet struct arm32_dma_range *ranges; 112129198Scognet int _nranges; 113166063Scognet struct bounce_zone *bounce_zone; 114244471Scognet /* 115244471Scognet * Most tags need one or two segments, and can use the local tagsegs 116244471Scognet * array. For tags with a larger limit, we'll allocate a bigger array 117244471Scognet * on first use. 118244471Scognet */ 119244471Scognet bus_dma_segment_t *segments; 120244471Scognet bus_dma_segment_t tagsegs[2]; 121129198Scognet}; 122129198Scognet 123166063Scognetstruct bounce_page { 124166063Scognet vm_offset_t vaddr; /* kva of bounce buffer */ 125166063Scognet bus_addr_t busaddr; /* Physical address */ 126166063Scognet vm_offset_t datavaddr; /* kva of client data */ 127246713Skib bus_addr_t dataaddr; /* client physical address */ 128166063Scognet bus_size_t datacount; /* client data count */ 129166063Scognet STAILQ_ENTRY(bounce_page) links; 130166063Scognet}; 131166063Scognet 132246713Skibstruct sync_list { 133246713Skib vm_offset_t vaddr; /* kva of bounce buffer */ 134246713Skib bus_addr_t busaddr; /* Physical address */ 135246713Skib bus_size_t datacount; /* client data count */ 136246713Skib}; 137246713Skib 138166063Scognetint busdma_swi_pending; 139166063Scognet 140166063Scognetstruct bounce_zone { 141166063Scognet STAILQ_ENTRY(bounce_zone) links; 142166063Scognet STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 143166063Scognet int total_bpages; 144166063Scognet int free_bpages; 145166063Scognet int reserved_bpages; 146166063Scognet int active_bpages; 147166063Scognet int total_bounced; 148166063Scognet int total_deferred; 149188403Scognet int map_count; 150166063Scognet bus_size_t alignment; 151166063Scognet bus_addr_t lowaddr; 152166063Scognet char zoneid[8]; 153166063Scognet char lowaddrid[20]; 154166063Scognet struct sysctl_ctx_list sysctl_tree; 155166063Scognet struct sysctl_oid *sysctl_tree_top; 156166063Scognet}; 157166063Scognet 158166063Scognetstatic struct mtx bounce_lock; 159166063Scognetstatic int total_bpages; 160166063Scognetstatic int busdma_zonecount; 161166063Scognetstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 162166063Scognet 163227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 164166063ScognetSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 165166063Scognet "Total bounce pages"); 166166063Scognet 167246713Skib#define DMAMAP_COHERENT 0x8 168244471Scognet#define DMAMAP_CACHE_ALIGNED 0x10 169246713Skib 170129198Scognetstruct bus_dmamap { 171166063Scognet struct bp_list bpages; 172166063Scognet int pagesneeded; 173166063Scognet int pagesreserved; 174135644Scognet bus_dma_tag_t dmat; 175246713Skib struct memdesc mem; 176135644Scognet int flags; 177166063Scognet STAILQ_ENTRY(bus_dmamap) links; 178166063Scognet bus_dmamap_callback_t *callback; 179166063Scognet void *callback_arg; 180246713Skib int sync_count; 181246713Skib struct sync_list *slist; 182129198Scognet}; 183129198Scognet 184166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 185166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 186166063Scognet 187146597Scognetstatic struct mtx busdma_mtx; 188146597Scognet 189146597ScognetMTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 190146597Scognet 191166063Scognetstatic void init_bounce_pages(void *dummy); 192166063Scognetstatic int alloc_bounce_zone(bus_dma_tag_t dmat); 193166063Scognetstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 194166063Scognetstatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 195166063Scognet int commit); 196166063Scognetstatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 197246713Skib vm_offset_t vaddr, bus_addr_t addr, 198246713Skib bus_size_t size); 199166063Scognetstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200166063Scognet 201166063Scognet/* Default tag, as most drivers provide no parent tag. */ 202166063Scognetbus_dma_tag_t arm_root_dma_tag; 203166063Scognet 204244473Scognet/* 205244473Scognet * ---------------------------------------------------------------------------- 206244473Scognet * Begin block of code useful to transplant to other implementations. 207244473Scognet */ 208244471Scognet 209244471Scognetstatic uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 210244471Scognet 211244471Scognetstatic busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 212244471Scognetstatic busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 213244471Scognet 214166063Scognet/* 215244471Scognet * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 216244471Scognet * It'll need platform-specific changes if this code is copied. 217244471Scognet */ 218244471Scognetstatic int 219244471Scognetdmamap_ctor(void *mem, int size, void *arg, int flags) 220244471Scognet{ 221244471Scognet bus_dmamap_t map; 222244471Scognet bus_dma_tag_t dmat; 223244471Scognet 224244471Scognet map = (bus_dmamap_t)mem; 225244471Scognet dmat = (bus_dma_tag_t)arg; 226244471Scognet 227244471Scognet dmat->map_count++; 228244471Scognet 229244471Scognet map->dmat = dmat; 230244471Scognet map->flags = 0; 231244471Scognet STAILQ_INIT(&map->bpages); 232244471Scognet 233244471Scognet return (0); 234244471Scognet} 235244471Scognet 236244471Scognet/* 237244471Scognet * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 238244471Scognet * It may need platform-specific changes if this code is copied . 239244471Scognet */ 240244471Scognetstatic void 241244471Scognetdmamap_dtor(void *mem, int size, void *arg) 242244471Scognet{ 243244471Scognet bus_dmamap_t map; 244244471Scognet 245244471Scognet map = (bus_dmamap_t)mem; 246244471Scognet 247244471Scognet map->dmat->map_count--; 248244471Scognet} 249244471Scognet 250244471Scognetstatic void 251244471Scognetbusdma_init(void *dummy) 252244471Scognet{ 253244471Scognet 254244471Scognet /* Create a cache of maps for bus_dmamap_create(). */ 255244471Scognet dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 256244471Scognet dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 257244471Scognet 258244471Scognet /* Create a cache of buffers in standard (cacheable) memory. */ 259244471Scognet standard_allocator = busdma_bufalloc_create("buffer", 260244471Scognet arm_dcache_align, /* minimum_alignment */ 261244471Scognet NULL, /* uma_alloc func */ 262244471Scognet NULL, /* uma_free func */ 263244471Scognet 0); /* uma_zcreate_flags */ 264244471Scognet 265244471Scognet /* 266244471Scognet * Create a cache of buffers in uncacheable memory, to implement the 267244471Scognet * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 268244471Scognet */ 269244471Scognet coherent_allocator = busdma_bufalloc_create("coherent", 270244471Scognet arm_dcache_align, /* minimum_alignment */ 271244471Scognet busdma_bufalloc_alloc_uncacheable, 272244471Scognet busdma_bufalloc_free_uncacheable, 273244471Scognet 0); /* uma_zcreate_flags */ 274244471Scognet} 275244471Scognet 276244471Scognet/* 277244471Scognet * This init historically used SI_SUB_VM, but now the init code requires 278244471Scognet * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 279244471Scognet * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 280244471Scognet * SI_SUB_KMEM and SI_ORDER_THIRD. 281244471Scognet */ 282244471ScognetSYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 283244471Scognet 284244473Scognet/* 285244473Scognet * End block of code useful to transplant to other implementations. 286244473Scognet * ---------------------------------------------------------------------------- 287244473Scognet */ 288244471Scognet 289244471Scognet/* 290166063Scognet * Return true if a match is made. 291166063Scognet * 292166063Scognet * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 293166063Scognet * 294166063Scognet * If paddr is within the bounds of the dma tag then call the filter callback 295166063Scognet * to check for a match, if there is no filter callback then assume a match. 296166063Scognet */ 297166063Scognetstatic int 298166063Scognetrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 299166063Scognet{ 300166063Scognet int retval; 301166063Scognet 302166063Scognet retval = 0; 303166063Scognet 304166063Scognet do { 305166063Scognet if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 306166063Scognet || ((paddr & (dmat->alignment - 1)) != 0)) 307166063Scognet && (dmat->filter == NULL 308166063Scognet || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 309166063Scognet retval = 1; 310166063Scognet 311166063Scognet dmat = dmat->parent; 312166063Scognet } while (retval == 0 && dmat != NULL); 313166063Scognet return (retval); 314166063Scognet} 315166063Scognet 316129198Scognet/* 317244471Scognet * This routine checks the exclusion zone constraints from a tag against the 318244471Scognet * physical RAM available on the machine. If a tag specifies an exclusion zone 319244471Scognet * but there's no RAM in that zone, then we avoid allocating resources to bounce 320244471Scognet * a request, and we can use any memory allocator (as opposed to needing 321244471Scognet * kmem_alloc_contig() just because it can allocate pages in an address range). 322244471Scognet * 323244471Scognet * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 324244471Scognet * same value on 32-bit architectures) as their lowaddr constraint, and we can't 325244471Scognet * possibly have RAM at an address higher than the highest address we can 326244471Scognet * express, so we take a fast out. 327129198Scognet */ 328137758Scognetstatic __inline int 329166063Scognet_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 330166063Scognet{ 331166063Scognet int i; 332244471Scognet 333244471Scognet if (lowaddr >= BUS_SPACE_MAXADDR) 334244471Scognet return (0); 335244471Scognet 336166063Scognet for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 337166063Scognet if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 338236991Simp || (lowaddr < phys_avail[i] && 339166063Scognet highaddr > phys_avail[i])) 340166063Scognet return (1); 341166063Scognet } 342166063Scognet return (0); 343166063Scognet} 344166063Scognet 345129198Scognetstatic __inline struct arm32_dma_range * 346129198Scognet_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 347129198Scognet bus_addr_t curaddr) 348129198Scognet{ 349129198Scognet struct arm32_dma_range *dr; 350129198Scognet int i; 351129198Scognet 352129198Scognet for (i = 0, dr = ranges; i < nranges; i++, dr++) { 353129198Scognet if (curaddr >= dr->dr_sysbase && 354129198Scognet round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 355129198Scognet return (dr); 356129198Scognet } 357129198Scognet 358129198Scognet return (NULL); 359129198Scognet} 360129198Scognet/* 361129198Scognet * Convenience function for manipulating driver locks from busdma (during 362129198Scognet * busdma_swi, for example). Drivers that don't provide their own locks 363129198Scognet * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 364129198Scognet * non-mutex locking scheme don't have to use this at all. 365129198Scognet */ 366129198Scognetvoid 367129198Scognetbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 368129198Scognet{ 369129198Scognet struct mtx *dmtx; 370129198Scognet 371129198Scognet dmtx = (struct mtx *)arg; 372129198Scognet switch (op) { 373129198Scognet case BUS_DMA_LOCK: 374129198Scognet mtx_lock(dmtx); 375129198Scognet break; 376129198Scognet case BUS_DMA_UNLOCK: 377129198Scognet mtx_unlock(dmtx); 378129198Scognet break; 379129198Scognet default: 380129198Scognet panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 381129198Scognet } 382129198Scognet} 383129198Scognet 384129198Scognet/* 385129198Scognet * dflt_lock should never get called. It gets put into the dma tag when 386129198Scognet * lockfunc == NULL, which is only valid if the maps that are associated 387129198Scognet * with the tag are meant to never be defered. 388129198Scognet * XXX Should have a way to identify which driver is responsible here. 389129198Scognet */ 390129198Scognetstatic void 391129198Scognetdflt_lock(void *arg, bus_dma_lock_op_t op) 392129198Scognet{ 393129198Scognet#ifdef INVARIANTS 394129198Scognet panic("driver error: busdma dflt_lock called"); 395129198Scognet#else 396129198Scognet printf("DRIVER_ERROR: busdma dflt_lock called\n"); 397129198Scognet#endif 398129198Scognet} 399129198Scognet 400129198Scognet/* 401129198Scognet * Allocate a device specific dma_tag. 402129198Scognet */ 403135644Scognet#define SEG_NB 1024 404135644Scognet 405129198Scognetint 406129198Scognetbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 407232356Sjhb bus_addr_t boundary, bus_addr_t lowaddr, 408129198Scognet bus_addr_t highaddr, bus_dma_filter_t *filter, 409129198Scognet void *filterarg, bus_size_t maxsize, int nsegments, 410129198Scognet bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 411129198Scognet void *lockfuncarg, bus_dma_tag_t *dmat) 412129198Scognet{ 413129198Scognet bus_dma_tag_t newtag; 414129198Scognet int error = 0; 415129198Scognet /* Return a NULL tag on failure */ 416129198Scognet *dmat = NULL; 417166063Scognet if (!parent) 418166063Scognet parent = arm_root_dma_tag; 419129198Scognet 420129198Scognet newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 421140313Scognet if (newtag == NULL) { 422143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 423143284Smux __func__, newtag, 0, error); 424129198Scognet return (ENOMEM); 425140313Scognet } 426129198Scognet 427129198Scognet newtag->parent = parent; 428244471Scognet newtag->alignment = alignment ? alignment : 1; 429129198Scognet newtag->boundary = boundary; 430129198Scognet newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 431129198Scognet newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 432129198Scognet newtag->filter = filter; 433129198Scognet newtag->filterarg = filterarg; 434129198Scognet newtag->maxsize = maxsize; 435129198Scognet newtag->nsegments = nsegments; 436129198Scognet newtag->maxsegsz = maxsegsz; 437129198Scognet newtag->flags = flags; 438129198Scognet newtag->ref_count = 1; /* Count ourself */ 439129198Scognet newtag->map_count = 0; 440129198Scognet newtag->ranges = bus_dma_get_range(); 441135644Scognet newtag->_nranges = bus_dma_get_range_nb(); 442129198Scognet if (lockfunc != NULL) { 443129198Scognet newtag->lockfunc = lockfunc; 444129198Scognet newtag->lockfuncarg = lockfuncarg; 445129198Scognet } else { 446129198Scognet newtag->lockfunc = dflt_lock; 447129198Scognet newtag->lockfuncarg = NULL; 448129198Scognet } 449244471Scognet /* 450244471Scognet * If all the segments we need fit into the local tagsegs array, set the 451244471Scognet * pointer now. Otherwise NULL the pointer and an array of segments 452244471Scognet * will be allocated later, on first use. We don't pre-allocate now 453244471Scognet * because some tags exist just to pass contraints to children in the 454244471Scognet * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 455244471Scognet * sure don't want to try to allocate an array for that. 456244471Scognet */ 457244471Scognet if (newtag->nsegments <= nitems(newtag->tagsegs)) 458244471Scognet newtag->segments = newtag->tagsegs; 459244471Scognet else 460244471Scognet newtag->segments = NULL; 461244471Scognet /* 462129198Scognet * Take into account any restrictions imposed by our parent tag 463129198Scognet */ 464129198Scognet if (parent != NULL) { 465232356Sjhb newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 466232356Sjhb newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 467134934Sscottl if (newtag->boundary == 0) 468134934Sscottl newtag->boundary = parent->boundary; 469134934Sscottl else if (parent->boundary != 0) 470232356Sjhb newtag->boundary = MIN(parent->boundary, 471134934Sscottl newtag->boundary); 472166063Scognet if ((newtag->filter != NULL) || 473166063Scognet ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 474166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 475129198Scognet if (newtag->filter == NULL) { 476129198Scognet /* 477129198Scognet * Short circuit looking at our parent directly 478129198Scognet * since we have encapsulated all of its information 479129198Scognet */ 480129198Scognet newtag->filter = parent->filter; 481129198Scognet newtag->filterarg = parent->filterarg; 482129198Scognet newtag->parent = parent->parent; 483129198Scognet } 484129198Scognet if (newtag->parent != NULL) 485129198Scognet atomic_add_int(&parent->ref_count, 1); 486129198Scognet } 487166063Scognet if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 488166063Scognet || newtag->alignment > 1) 489166063Scognet newtag->flags |= BUS_DMA_COULD_BOUNCE; 490129198Scognet 491166063Scognet if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 492166063Scognet (flags & BUS_DMA_ALLOCNOW) != 0) { 493166063Scognet struct bounce_zone *bz; 494166063Scognet 495166063Scognet /* Must bounce */ 496166063Scognet 497166063Scognet if ((error = alloc_bounce_zone(newtag)) != 0) { 498166063Scognet free(newtag, M_DEVBUF); 499166063Scognet return (error); 500166063Scognet } 501166063Scognet bz = newtag->bounce_zone; 502166063Scognet 503166063Scognet if (ptoa(bz->total_bpages) < maxsize) { 504166063Scognet int pages; 505166063Scognet 506166063Scognet pages = atop(maxsize) - bz->total_bpages; 507166063Scognet 508166063Scognet /* Add pages to our bounce pool */ 509166063Scognet if (alloc_bounce_pages(newtag, pages) < pages) 510166063Scognet error = ENOMEM; 511166063Scognet } 512166063Scognet /* Performed initial allocation */ 513166063Scognet newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 514170502Scognet } else 515170502Scognet newtag->bounce_zone = NULL; 516166063Scognet if (error != 0) 517166063Scognet free(newtag, M_DEVBUF); 518166063Scognet else 519166063Scognet *dmat = newtag; 520143294Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 521143284Smux __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 522140313Scognet 523129198Scognet return (error); 524129198Scognet} 525129198Scognet 526129198Scognetint 527129198Scognetbus_dma_tag_destroy(bus_dma_tag_t dmat) 528129198Scognet{ 529140680Scognet#ifdef KTR 530140313Scognet bus_dma_tag_t dmat_copy = dmat; 531140680Scognet#endif 532140313Scognet 533129198Scognet if (dmat != NULL) { 534244471Scognet 535129198Scognet if (dmat->map_count != 0) 536129198Scognet return (EBUSY); 537129198Scognet 538129198Scognet while (dmat != NULL) { 539129198Scognet bus_dma_tag_t parent; 540129198Scognet 541129198Scognet parent = dmat->parent; 542129198Scognet atomic_subtract_int(&dmat->ref_count, 1); 543129198Scognet if (dmat->ref_count == 0) { 544244471Scognet if (dmat->segments != NULL && 545244471Scognet dmat->segments != dmat->tagsegs) 546240177Sjhb free(dmat->segments, M_DEVBUF); 547129198Scognet free(dmat, M_DEVBUF); 548129198Scognet /* 549129198Scognet * Last reference count, so 550129198Scognet * release our reference 551129198Scognet * count on our parent. 552129198Scognet */ 553129198Scognet dmat = parent; 554129198Scognet } else 555129198Scognet dmat = NULL; 556129198Scognet } 557129198Scognet } 558143294Smux CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 559140313Scognet 560129198Scognet return (0); 561129198Scognet} 562129198Scognet 563166063Scognet#include <sys/kdb.h> 564129198Scognet/* 565129198Scognet * Allocate a handle for mapping from kva/uva/physical 566129198Scognet * address space into bus device space. 567129198Scognet */ 568129198Scognetint 569129198Scognetbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 570129198Scognet{ 571246713Skib struct sync_list *slist; 572244471Scognet bus_dmamap_t map; 573140313Scognet int error = 0; 574129198Scognet 575246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 576246713Skib if (slist == NULL) 577246713Skib return (ENOMEM); 578246713Skib 579244575Scognet map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 580244471Scognet *mapp = map; 581246713Skib if (map == NULL) { 582246713Skib free(slist, M_DEVBUF); 583244575Scognet return (ENOMEM); 584246713Skib } 585240177Sjhb 586244471Scognet /* 587244471Scognet * If the tag's segments haven't been allocated yet we need to do it 588244471Scognet * now, because we can't sleep for resources at map load time. 589244471Scognet */ 590244575Scognet if (dmat->segments == NULL) { 591244471Scognet dmat->segments = malloc(dmat->nsegments * 592244575Scognet sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 593244575Scognet if (dmat->segments == NULL) { 594246713Skib free(slist, M_DEVBUF); 595244575Scognet uma_zfree(dmamap_zone, map); 596244575Scognet *mapp = NULL; 597244575Scognet return (ENOMEM); 598244575Scognet } 599244575Scognet } 600129198Scognet 601166063Scognet /* 602166063Scognet * Bouncing might be required if the driver asks for an active 603166063Scognet * exclusion region, a data alignment that is stricter than 1, and/or 604166063Scognet * an active address boundary. 605166063Scognet */ 606166063Scognet if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 607166063Scognet 608166063Scognet /* Must bounce */ 609166063Scognet struct bounce_zone *bz; 610166063Scognet int maxpages; 611166063Scognet 612166063Scognet if (dmat->bounce_zone == NULL) { 613166063Scognet if ((error = alloc_bounce_zone(dmat)) != 0) { 614246713Skib free(slist, M_DEVBUF); 615244471Scognet uma_zfree(dmamap_zone, map); 616166063Scognet *mapp = NULL; 617166063Scognet return (error); 618166063Scognet } 619166063Scognet } 620166063Scognet bz = dmat->bounce_zone; 621166063Scognet 622166063Scognet /* Initialize the new map */ 623166063Scognet STAILQ_INIT(&((*mapp)->bpages)); 624166063Scognet 625166063Scognet /* 626166063Scognet * Attempt to add pages to our pool on a per-instance 627166063Scognet * basis up to a sane limit. 628166063Scognet */ 629166063Scognet maxpages = MAX_BPAGES; 630166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 631188403Scognet || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 632166063Scognet int pages; 633166063Scognet 634166063Scognet pages = MAX(atop(dmat->maxsize), 1); 635166063Scognet pages = MIN(maxpages - bz->total_bpages, pages); 636166063Scognet pages = MAX(pages, 1); 637166063Scognet if (alloc_bounce_pages(dmat, pages) < pages) 638166063Scognet error = ENOMEM; 639166063Scognet 640166063Scognet if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 641166063Scognet if (error == 0) 642166063Scognet dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 643166063Scognet } else { 644166063Scognet error = 0; 645166063Scognet } 646166063Scognet } 647188403Scognet bz->map_count++; 648166063Scognet } 649246713Skib map->sync_count = 0; 650246713Skib map->slist = slist; 651143294Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 652143284Smux __func__, dmat, dmat->flags, error); 653140313Scognet 654129198Scognet return (0); 655129198Scognet} 656129198Scognet 657129198Scognet/* 658129198Scognet * Destroy a handle for mapping from kva/uva/physical 659129198Scognet * address space into bus device space. 660129198Scognet */ 661129198Scognetint 662129198Scognetbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 663129198Scognet{ 664135644Scognet 665246713Skib if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 666166063Scognet CTR3(KTR_BUSDMA, "%s: tag %p error %d", 667166063Scognet __func__, dmat, EBUSY); 668166063Scognet return (EBUSY); 669166063Scognet } 670246713Skib free(map->slist, M_DEVBUF); 671244471Scognet uma_zfree(dmamap_zone, map); 672188403Scognet if (dmat->bounce_zone) 673188403Scognet dmat->bounce_zone->map_count--; 674143294Smux CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 675129198Scognet return (0); 676129198Scognet} 677129198Scognet 678129198Scognet/* 679244471Scognet * Allocate a piece of memory that can be efficiently mapped into bus device 680244471Scognet * space based on the constraints listed in the dma tag. Returns a pointer to 681244471Scognet * the allocated memory, and a pointer to an associated bus_dmamap. 682129198Scognet */ 683129198Scognetint 684244471Scognetbus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 685129198Scognet bus_dmamap_t *mapp) 686129198Scognet{ 687246713Skib struct sync_list *slist; 688244471Scognet void * vaddr; 689244471Scognet struct busdma_bufzone *bufzone; 690244471Scognet busdma_bufalloc_t ba; 691244471Scognet bus_dmamap_t map; 692129198Scognet int mflags; 693244471Scognet vm_memattr_t memattr; 694129198Scognet 695129198Scognet if (flags & BUS_DMA_NOWAIT) 696129198Scognet mflags = M_NOWAIT; 697129198Scognet else 698129198Scognet mflags = M_WAITOK; 699244471Scognet /* 700244471Scognet * If the tag's segments haven't been allocated yet we need to do it 701244471Scognet * now, because we can't sleep for resources at map load time. 702244471Scognet */ 703244471Scognet if (dmat->segments == NULL) 704244471Scognet dmat->segments = malloc(dmat->nsegments * 705244471Scognet sizeof(*dmat->segments), M_DEVBUF, mflags); 706244471Scognet 707246713Skib slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 708246713Skib if (slist == NULL) 709246713Skib return (ENOMEM); 710244471Scognet map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 711246713Skib if (map == NULL) { 712246713Skib free(slist, M_DEVBUF); 713244471Scognet return (ENOMEM); 714246713Skib } 715244471Scognet if (flags & BUS_DMA_COHERENT) { 716244471Scognet memattr = VM_MEMATTR_UNCACHEABLE; 717244471Scognet ba = coherent_allocator; 718244471Scognet map->flags |= DMAMAP_COHERENT; 719244471Scognet } else { 720244471Scognet memattr = VM_MEMATTR_DEFAULT; 721244471Scognet ba = standard_allocator; 722240177Sjhb } 723244471Scognet /* All buffers we allocate are cache-aligned. */ 724244471Scognet map->flags |= DMAMAP_CACHE_ALIGNED; 725244471Scognet 726129198Scognet if (flags & BUS_DMA_ZERO) 727129198Scognet mflags |= M_ZERO; 728129198Scognet 729244471Scognet /* 730244471Scognet * Try to find a bufzone in the allocator that holds a cache of buffers 731244471Scognet * of the right size for this request. If the buffer is too big to be 732244471Scognet * held in the allocator cache, this returns NULL. 733244471Scognet */ 734244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 735244471Scognet 736244471Scognet /* 737244471Scognet * Allocate the buffer from the uma(9) allocator if... 738244471Scognet * - It's small enough to be in the allocator (bufzone not NULL). 739244471Scognet * - The alignment constraint isn't larger than the allocation size 740244471Scognet * (the allocator aligns buffers to their size boundaries). 741244471Scognet * - There's no need to handle lowaddr/highaddr exclusion zones. 742244471Scognet * else allocate non-contiguous pages if... 743244471Scognet * - The page count that could get allocated doesn't exceed nsegments. 744244471Scognet * - The alignment constraint isn't larger than a page boundary. 745244471Scognet * - There are no boundary-crossing constraints. 746244471Scognet * else allocate a block of contiguous pages because one or more of the 747244471Scognet * constraints is something that only the contig allocator can fulfill. 748244471Scognet */ 749244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 750244471Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 751244471Scognet vaddr = uma_zalloc(bufzone->umazone, mflags); 752318977Shselasky } else if (dmat->nsegments >= 753318977Shselasky howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) && 754318977Shselasky dmat->alignment <= PAGE_SIZE && 755318977Shselasky (dmat->boundary % PAGE_SIZE) == 0) { 756254025Sjeff vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 757244471Scognet mflags, 0, dmat->lowaddr, memattr); 758244471Scognet } else { 759254025Sjeff vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 760244471Scognet mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 761244471Scognet memattr); 762135644Scognet } 763244471Scognet if (vaddr == NULL) { 764246713Skib free(slist, M_DEVBUF); 765244471Scognet uma_zfree(dmamap_zone, map); 766244471Scognet map = NULL; 767246713Skib } else { 768246713Skib map->slist = slist; 769246713Skib map->sync_count = 0; 770129198Scognet } 771244471Scognet *vaddrp = vaddr; 772244471Scognet *mapp = map; 773244471Scognet 774244471Scognet return (vaddr == NULL ? ENOMEM : 0); 775129198Scognet} 776129198Scognet 777129198Scognet/* 778244471Scognet * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 779244471Scognet * its associated map. 780129198Scognet */ 781129198Scognetvoid 782129198Scognetbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 783129198Scognet{ 784244471Scognet struct busdma_bufzone *bufzone; 785244471Scognet busdma_bufalloc_t ba; 786244471Scognet 787244471Scognet if (map->flags & DMAMAP_COHERENT) 788244471Scognet ba = coherent_allocator; 789246713Skib else 790244471Scognet ba = standard_allocator; 791244471Scognet 792246713Skib free(map->slist, M_DEVBUF); 793294671Sian uma_zfree(dmamap_zone, map); 794244471Scognet 795244471Scognet bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 796244471Scognet 797244471Scognet if (bufzone != NULL && dmat->alignment <= bufzone->size && 798166063Scognet !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 799244471Scognet uma_zfree(bufzone->umazone, vaddr); 800244471Scognet else 801254025Sjeff kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 802129198Scognet} 803129198Scognet 804246713Skibstatic void 805246713Skib_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 806246713Skib bus_size_t buflen, int flags) 807246713Skib{ 808246713Skib bus_addr_t curaddr; 809246713Skib bus_size_t sgsize; 810246713Skib 811259337Sian if (map->pagesneeded == 0) { 812246713Skib CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 813246713Skib dmat->lowaddr, dmat->boundary, dmat->alignment); 814246713Skib CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 815246713Skib map, map->pagesneeded); 816246713Skib /* 817246713Skib * Count the number of bounce pages 818246713Skib * needed in order to complete this transfer 819246713Skib */ 820246713Skib curaddr = buf; 821246713Skib while (buflen != 0) { 822246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 823246713Skib if (run_filter(dmat, curaddr) != 0) { 824246713Skib sgsize = MIN(sgsize, PAGE_SIZE); 825246713Skib map->pagesneeded++; 826246713Skib } 827246713Skib curaddr += sgsize; 828246713Skib buflen -= sgsize; 829246713Skib } 830246713Skib CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 831246713Skib } 832246713Skib} 833246713Skib 834246713Skibstatic void 835191011Skib_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 836191011Skib void *buf, bus_size_t buflen, int flags) 837166063Scognet{ 838166063Scognet vm_offset_t vaddr; 839166063Scognet vm_offset_t vendaddr; 840166063Scognet bus_addr_t paddr; 841166063Scognet 842259337Sian if (map->pagesneeded == 0) { 843185494Sstas CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 844185494Sstas dmat->lowaddr, dmat->boundary, dmat->alignment); 845170406Scognet CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 846170406Scognet map, map->pagesneeded); 847166063Scognet /* 848166063Scognet * Count the number of bounce pages 849166063Scognet * needed in order to complete this transfer 850166063Scognet */ 851166063Scognet vaddr = trunc_page((vm_offset_t)buf); 852166063Scognet vendaddr = (vm_offset_t)buf + buflen; 853166063Scognet 854166063Scognet while (vaddr < vendaddr) { 855246713Skib if (__predict_true(pmap == kernel_pmap)) 856191438Sjhb paddr = pmap_kextract(vaddr); 857191438Sjhb else 858191011Skib paddr = pmap_extract(pmap, vaddr); 859246713Skib if (run_filter(dmat, paddr) != 0) 860166063Scognet map->pagesneeded++; 861166063Scognet vaddr += PAGE_SIZE; 862166063Scognet } 863166063Scognet CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 864166063Scognet } 865246713Skib} 866166063Scognet 867246713Skibstatic int 868246713Skib_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 869246713Skib{ 870246713Skib 871166063Scognet /* Reserve Necessary Bounce Pages */ 872246713Skib mtx_lock(&bounce_lock); 873246713Skib if (flags & BUS_DMA_NOWAIT) { 874246713Skib if (reserve_bounce_pages(dmat, map, 0) != 0) { 875246713Skib mtx_unlock(&bounce_lock); 876246713Skib return (ENOMEM); 877166063Scognet } 878246713Skib } else { 879246713Skib if (reserve_bounce_pages(dmat, map, 1) != 0) { 880246713Skib /* Queue us for resources */ 881246713Skib STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 882246713Skib mtx_unlock(&bounce_lock); 883246713Skib return (EINPROGRESS); 884246713Skib } 885166063Scognet } 886246713Skib mtx_unlock(&bounce_lock); 887166063Scognet 888166063Scognet return (0); 889166063Scognet} 890166063Scognet 891129198Scognet/* 892246713Skib * Add a single contiguous physical range to the segment list. 893246713Skib */ 894246713Skibstatic int 895246713Skib_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 896246713Skib bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 897246713Skib{ 898246713Skib bus_addr_t baddr, bmask; 899246713Skib int seg; 900246713Skib 901246713Skib /* 902246713Skib * Make sure we don't cross any boundaries. 903246713Skib */ 904246713Skib bmask = ~(dmat->boundary - 1); 905246713Skib if (dmat->boundary > 0) { 906246713Skib baddr = (curaddr + dmat->boundary) & bmask; 907246713Skib if (sgsize > (baddr - curaddr)) 908246713Skib sgsize = (baddr - curaddr); 909246713Skib } 910246713Skib if (dmat->ranges) { 911246713Skib struct arm32_dma_range *dr; 912246713Skib 913246713Skib dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 914246713Skib curaddr); 915246713Skib if (dr == NULL) 916246881Sian return (0); 917246713Skib /* 918246713Skib * In a valid DMA range. Translate the physical 919246713Skib * memory address to an address in the DMA window. 920246713Skib */ 921246713Skib curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 922246713Skib 923246713Skib } 924246713Skib 925246713Skib seg = *segp; 926246713Skib /* 927246713Skib * Insert chunk into a segment, coalescing with 928246713Skib * the previous segment if possible. 929246713Skib */ 930246713Skib if (seg >= 0 && 931246713Skib curaddr == segs[seg].ds_addr + segs[seg].ds_len && 932246713Skib (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 933246713Skib (dmat->boundary == 0 || 934246713Skib (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 935246713Skib segs[seg].ds_len += sgsize; 936246713Skib } else { 937246713Skib if (++seg >= dmat->nsegments) 938246881Sian return (0); 939246713Skib segs[seg].ds_addr = curaddr; 940246713Skib segs[seg].ds_len = sgsize; 941246713Skib } 942246713Skib *segp = seg; 943246881Sian return (sgsize); 944246713Skib} 945246713Skib 946246713Skib/* 947246713Skib * Utility function to load a physical buffer. segp contains 948246713Skib * the starting segment on entrace, and the ending segment on exit. 949246713Skib */ 950246713Skibint 951246713Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 952246713Skib bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 953246713Skib{ 954246713Skib bus_size_t sgsize; 955246713Skib bus_addr_t curaddr; 956246713Skib int error; 957246713Skib 958246713Skib if (segs == NULL) 959246713Skib segs = dmat->segments; 960246713Skib 961246713Skib if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 962246713Skib _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 963246713Skib if (map->pagesneeded != 0) { 964246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 965246713Skib if (error) 966246713Skib return (error); 967246713Skib } 968246713Skib } 969246713Skib 970246713Skib while (buflen > 0) { 971246713Skib curaddr = buf; 972246713Skib sgsize = MIN(buflen, dmat->maxsegsz); 973246713Skib if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 974246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 975246713Skib sgsize = MIN(sgsize, PAGE_SIZE); 976246713Skib curaddr = add_bounce_page(dmat, map, 0, curaddr, 977246713Skib sgsize); 978246713Skib } 979246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 980246713Skib segp); 981246713Skib if (sgsize == 0) 982246713Skib break; 983246713Skib buf += sgsize; 984246713Skib buflen -= sgsize; 985246713Skib } 986246713Skib 987246713Skib /* 988246713Skib * Did we fit? 989246713Skib */ 990246713Skib if (buflen != 0) { 991246713Skib _bus_dmamap_unload(dmat, map); 992246713Skib return (EFBIG); /* XXX better return value here? */ 993246713Skib } 994246713Skib return (0); 995246713Skib} 996259510Skib 997259510Skibint 998259510Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 999259510Skib struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1000259510Skib bus_dma_segment_t *segs, int *segp) 1001259510Skib{ 1002259510Skib 1003259510Skib return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1004259510Skib segs, segp)); 1005259510Skib} 1006259510Skib 1007246713Skib/* 1008246713Skib * Utility function to load a linear buffer. segp contains 1009129198Scognet * the starting segment on entrance, and the ending segment on exit. 1010129198Scognet */ 1011246713Skibint 1012246713Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1013246713Skib bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1014246713Skib int *segp) 1015129198Scognet{ 1016129198Scognet bus_size_t sgsize; 1017246713Skib bus_addr_t curaddr; 1018246713Skib struct sync_list *sl; 1019129198Scognet vm_offset_t vaddr = (vm_offset_t)buf; 1020129198Scognet int error = 0; 1021129198Scognet 1022246713Skib if (segs == NULL) 1023246713Skib segs = dmat->segments; 1024246713Skib if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1025246713Skib map->flags |= DMAMAP_CACHE_ALIGNED; 1026129198Scognet 1027166063Scognet if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1028246713Skib _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1029246713Skib if (map->pagesneeded != 0) { 1030246713Skib error = _bus_dmamap_reserve_pages(dmat, map, flags); 1031246713Skib if (error) 1032246713Skib return (error); 1033246713Skib } 1034166063Scognet } 1035140313Scognet CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1036140313Scognet "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1037140313Scognet 1038246713Skib while (buflen > 0) { 1039129198Scognet /* 1040129198Scognet * Get the physical address for this segment. 1041129198Scognet */ 1042246713Skib if (__predict_true(pmap == kernel_pmap)) { 1043246158Skib curaddr = pmap_kextract(vaddr); 1044129198Scognet } else { 1045129198Scognet curaddr = pmap_extract(pmap, vaddr); 1046135644Scognet map->flags &= ~DMAMAP_COHERENT; 1047129198Scognet } 1048129198Scognet 1049129198Scognet /* 1050129198Scognet * Compute the segment size, and adjust counts. 1051129198Scognet */ 1052129198Scognet sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 1053170086Syongari if (sgsize > dmat->maxsegsz) 1054170086Syongari sgsize = dmat->maxsegsz; 1055129198Scognet if (buflen < sgsize) 1056129198Scognet sgsize = buflen; 1057129198Scognet 1058166063Scognet if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1059246713Skib map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1060246713Skib curaddr = add_bounce_page(dmat, map, vaddr, curaddr, 1061246713Skib sgsize); 1062137760Scognet } else { 1063246713Skib sl = &map->slist[map->sync_count - 1]; 1064246713Skib if (map->sync_count == 0 || 1065246713Skib vaddr != sl->vaddr + sl->datacount) { 1066246713Skib if (++map->sync_count > dmat->nsegments) 1067246713Skib goto cleanup; 1068246713Skib sl++; 1069246713Skib sl->vaddr = vaddr; 1070246713Skib sl->datacount = sgsize; 1071246713Skib sl->busaddr = curaddr; 1072246713Skib } else 1073246713Skib sl->datacount += sgsize; 1074129198Scognet } 1075246713Skib sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1076246713Skib segp); 1077246713Skib if (sgsize == 0) 1078135644Scognet break; 1079129198Scognet vaddr += sgsize; 1080129198Scognet buflen -= sgsize; 1081129198Scognet } 1082129198Scognet 1083246713Skibcleanup: 1084129198Scognet /* 1085129198Scognet * Did we fit? 1086129198Scognet */ 1087246713Skib if (buflen != 0) { 1088246713Skib _bus_dmamap_unload(dmat, map); 1089246713Skib return (EFBIG); /* XXX better return value here? */ 1090246713Skib } 1091246713Skib return (0); 1092129198Scognet} 1093129198Scognet 1094246713Skibvoid 1095246713Skib__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 1096246713Skib struct memdesc *mem, bus_dmamap_callback_t *callback, 1097246713Skib void *callback_arg) 1098140682Scognet{ 1099140682Scognet 1100143671Sjmg KASSERT(dmat != NULL, ("dmatag is NULL")); 1101143671Sjmg KASSERT(map != NULL, ("dmamap is NULL")); 1102246713Skib map->mem = *mem; 1103166063Scognet map->callback = callback; 1104166063Scognet map->callback_arg = callback_arg; 1105140682Scognet} 1106140682Scognet 1107246713Skibbus_dma_segment_t * 1108246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1109246713Skib bus_dma_segment_t *segs, int nsegs, int error) 1110129198Scognet{ 1111129198Scognet 1112246713Skib if (segs == NULL) 1113246713Skib segs = dmat->segments; 1114246713Skib return (segs); 1115129198Scognet} 1116129198Scognet 1117129198Scognet/* 1118135644Scognet * Release the mapping held by map. 1119129198Scognet */ 1120129198Scognetvoid 1121143655Sjmg_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1122150893Scognet{ 1123166063Scognet struct bounce_page *bpage; 1124166063Scognet 1125166063Scognet while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1126166063Scognet STAILQ_REMOVE_HEAD(&map->bpages, links); 1127166063Scognet free_bounce_page(dmat, bpage); 1128166063Scognet } 1129246713Skib map->sync_count = 0; 1130129198Scognet return; 1131129198Scognet} 1132129198Scognet 1133169761Scognetstatic void 1134246713Skibbus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1135246713Skib int bufaligned) 1136135644Scognet{ 1137166063Scognet char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1138234561Smarius register_t s; 1139236991Simp int partial; 1140135644Scognet 1141171890Scognet if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1142246713Skib cpu_dcache_wb_range(buf, len); 1143246713Skib cpu_l2cache_wb_range(buf, len); 1144171623Scognet } 1145244471Scognet 1146244471Scognet /* 1147244471Scognet * If the caller promises the buffer is properly aligned to a cache line 1148244471Scognet * (even if the call parms make it look like it isn't) we can avoid 1149244471Scognet * attempting to preserve the non-DMA part of the cache line in the 1150244471Scognet * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1151244471Scognet * 1152244471Scognet * This covers the case of mbufs, where we know how they're aligned and 1153244471Scognet * know the CPU doesn't touch the header in front of the DMA data area 1154244471Scognet * during the IO, but it may have touched it right before invoking the 1155244471Scognet * sync, so a PREREAD writeback is required. 1156244471Scognet * 1157244471Scognet * It also handles buffers we created in bus_dmamem_alloc(), which are 1158244471Scognet * always aligned and padded to cache line size even if the IO length 1159244471Scognet * isn't a multiple of cache line size. In this case the PREREAD 1160244471Scognet * writeback probably isn't required, but it's harmless. 1161244471Scognet */ 1162234561Smarius partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1163244471Scognet 1164171623Scognet if (op & BUS_DMASYNC_PREREAD) { 1165234561Smarius if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1166246713Skib cpu_dcache_inv_range(buf, len); 1167246713Skib cpu_l2cache_inv_range(buf, len); 1168171890Scognet } else { 1169246713Skib cpu_dcache_wbinv_range(buf, len); 1170246713Skib cpu_l2cache_wbinv_range(buf, len); 1171171890Scognet } 1172171623Scognet } 1173166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1174244471Scognet if (partial && !bufaligned) { 1175234561Smarius s = intr_disable(); 1176246713Skib if (buf & arm_dcache_align_mask) 1177246713Skib memcpy(_tmp_cl, (void *)(buf & 1178234561Smarius ~arm_dcache_align_mask), 1179246713Skib buf & arm_dcache_align_mask); 1180246713Skib if ((buf + len) & arm_dcache_align_mask) 1181236991Simp memcpy(_tmp_clend, 1182246713Skib (void *)(buf + len), 1183246713Skib arm_dcache_align - 1184246713Skib ((buf + len) & arm_dcache_align_mask)); 1185171623Scognet } 1186246713Skib cpu_dcache_inv_range(buf, len); 1187246713Skib cpu_l2cache_inv_range(buf, len); 1188244471Scognet if (partial && !bufaligned) { 1189246713Skib if (buf & arm_dcache_align_mask) 1190246713Skib memcpy((void *)(buf & 1191236991Simp ~arm_dcache_align_mask), _tmp_cl, 1192246713Skib buf & arm_dcache_align_mask); 1193246713Skib if ((buf + len) & arm_dcache_align_mask) 1194246713Skib memcpy((void *)(buf + len), 1195236991Simp _tmp_clend, arm_dcache_align - 1196246713Skib ((buf + len) & arm_dcache_align_mask)); 1197234561Smarius intr_restore(s); 1198234561Smarius } 1199135644Scognet } 1200135644Scognet} 1201135644Scognet 1202166063Scognetstatic void 1203166063Scognet_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1204166063Scognet{ 1205166063Scognet struct bounce_page *bpage; 1206166063Scognet 1207166063Scognet STAILQ_FOREACH(bpage, &map->bpages, links) { 1208166063Scognet if (op & BUS_DMASYNC_PREWRITE) { 1209246713Skib if (bpage->datavaddr != 0) 1210259335Sian bcopy((void *)bpage->datavaddr, 1211259335Sian (void *)bpage->vaddr, bpage->datacount); 1212246713Skib else 1213246713Skib physcopyout(bpage->dataaddr, 1214259335Sian (void *)bpage->vaddr,bpage->datacount); 1215259335Sian cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1216259335Sian cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1217187911Sthompsa dmat->bounce_zone->total_bounced++; 1218166063Scognet } 1219166063Scognet if (op & BUS_DMASYNC_POSTREAD) { 1220259335Sian cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1221259335Sian cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1222246713Skib if (bpage->datavaddr != 0) 1223259335Sian bcopy((void *)bpage->vaddr, 1224246713Skib (void *)bpage->datavaddr, bpage->datacount); 1225246713Skib else 1226259335Sian physcopyin((void *)bpage->vaddr, 1227246713Skib bpage->dataaddr, bpage->datacount); 1228187911Sthompsa dmat->bounce_zone->total_bounced++; 1229166063Scognet } 1230166063Scognet } 1231166063Scognet} 1232166063Scognet 1233129198Scognetvoid 1234143655Sjmg_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1235129198Scognet{ 1236246713Skib struct sync_list *sl, *end; 1237244471Scognet int bufaligned; 1238244471Scognet 1239159107Scognet if (op == BUS_DMASYNC_POSTWRITE) 1240129198Scognet return; 1241244471Scognet if (map->flags & DMAMAP_COHERENT) 1242244471Scognet goto drain; 1243166063Scognet if (STAILQ_FIRST(&map->bpages)) 1244166063Scognet _bus_dmamap_sync_bp(dmat, map, op); 1245143294Smux CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1246244471Scognet bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1247246713Skib if (map->sync_count) { 1248246713Skib end = &map->slist[map->sync_count]; 1249246713Skib for (sl = &map->slist[0]; sl != end; sl++) 1250246713Skib bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, 1251244471Scognet bufaligned); 1252129198Scognet } 1253244471Scognet 1254244471Scognetdrain: 1255244471Scognet 1256135644Scognet cpu_drain_writebuf(); 1257129198Scognet} 1258166063Scognet 1259166063Scognetstatic void 1260166063Scognetinit_bounce_pages(void *dummy __unused) 1261166063Scognet{ 1262166063Scognet 1263166063Scognet total_bpages = 0; 1264166063Scognet STAILQ_INIT(&bounce_zone_list); 1265166063Scognet STAILQ_INIT(&bounce_map_waitinglist); 1266166063Scognet STAILQ_INIT(&bounce_map_callbacklist); 1267166063Scognet mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1268166063Scognet} 1269166063ScognetSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1270166063Scognet 1271166063Scognetstatic struct sysctl_ctx_list * 1272166063Scognetbusdma_sysctl_tree(struct bounce_zone *bz) 1273166063Scognet{ 1274166063Scognet return (&bz->sysctl_tree); 1275166063Scognet} 1276166063Scognet 1277166063Scognetstatic struct sysctl_oid * 1278166063Scognetbusdma_sysctl_tree_top(struct bounce_zone *bz) 1279166063Scognet{ 1280166063Scognet return (bz->sysctl_tree_top); 1281166063Scognet} 1282166063Scognet 1283166063Scognetstatic int 1284166063Scognetalloc_bounce_zone(bus_dma_tag_t dmat) 1285166063Scognet{ 1286166063Scognet struct bounce_zone *bz; 1287166063Scognet 1288166063Scognet /* Check to see if we already have a suitable zone */ 1289166063Scognet STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1290166063Scognet if ((dmat->alignment <= bz->alignment) 1291166063Scognet && (dmat->lowaddr >= bz->lowaddr)) { 1292166063Scognet dmat->bounce_zone = bz; 1293166063Scognet return (0); 1294166063Scognet } 1295166063Scognet } 1296166063Scognet 1297166063Scognet if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1298166063Scognet M_NOWAIT | M_ZERO)) == NULL) 1299166063Scognet return (ENOMEM); 1300166063Scognet 1301166063Scognet STAILQ_INIT(&bz->bounce_page_list); 1302166063Scognet bz->free_bpages = 0; 1303166063Scognet bz->reserved_bpages = 0; 1304166063Scognet bz->active_bpages = 0; 1305166063Scognet bz->lowaddr = dmat->lowaddr; 1306191438Sjhb bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1307188403Scognet bz->map_count = 0; 1308166063Scognet snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1309166063Scognet busdma_zonecount++; 1310166063Scognet snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1311166063Scognet STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1312166063Scognet dmat->bounce_zone = bz; 1313166063Scognet 1314166063Scognet sysctl_ctx_init(&bz->sysctl_tree); 1315166063Scognet bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1316166063Scognet SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1317166063Scognet CTLFLAG_RD, 0, ""); 1318166063Scognet if (bz->sysctl_tree_top == NULL) { 1319166063Scognet sysctl_ctx_free(&bz->sysctl_tree); 1320166063Scognet return (0); /* XXX error code? */ 1321166063Scognet } 1322166063Scognet 1323166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1324166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1325166063Scognet "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1326166063Scognet "Total bounce pages"); 1327166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1328166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1329166063Scognet "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1330166063Scognet "Free bounce pages"); 1331166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1332166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1333166063Scognet "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1334166063Scognet "Reserved bounce pages"); 1335166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1336166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1337166063Scognet "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1338166063Scognet "Active bounce pages"); 1339166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1340166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1341166063Scognet "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1342166063Scognet "Total bounce requests"); 1343166063Scognet SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1344166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1345166063Scognet "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1346166063Scognet "Total bounce requests that were deferred"); 1347166063Scognet SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1348166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1349166063Scognet "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1350273736Shselasky SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1351166063Scognet SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1352273736Shselasky "alignment", CTLFLAG_RD, &bz->alignment, ""); 1353166063Scognet 1354166063Scognet return (0); 1355166063Scognet} 1356166063Scognet 1357166063Scognetstatic int 1358166063Scognetalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1359166063Scognet{ 1360166063Scognet struct bounce_zone *bz; 1361166063Scognet int count; 1362166063Scognet 1363166063Scognet bz = dmat->bounce_zone; 1364166063Scognet count = 0; 1365166063Scognet while (numpages > 0) { 1366166063Scognet struct bounce_page *bpage; 1367166063Scognet 1368166063Scognet bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1369166063Scognet M_NOWAIT | M_ZERO); 1370166063Scognet 1371166063Scognet if (bpage == NULL) 1372166063Scognet break; 1373166063Scognet bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1374166063Scognet M_NOWAIT, 0ul, 1375166063Scognet bz->lowaddr, 1376166063Scognet PAGE_SIZE, 1377191438Sjhb 0); 1378166063Scognet if (bpage->vaddr == 0) { 1379166063Scognet free(bpage, M_DEVBUF); 1380166063Scognet break; 1381166063Scognet } 1382166063Scognet bpage->busaddr = pmap_kextract(bpage->vaddr); 1383166063Scognet mtx_lock(&bounce_lock); 1384166063Scognet STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1385166063Scognet total_bpages++; 1386166063Scognet bz->total_bpages++; 1387166063Scognet bz->free_bpages++; 1388166063Scognet mtx_unlock(&bounce_lock); 1389166063Scognet count++; 1390166063Scognet numpages--; 1391166063Scognet } 1392166063Scognet return (count); 1393166063Scognet} 1394166063Scognet 1395166063Scognetstatic int 1396166063Scognetreserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1397166063Scognet{ 1398166063Scognet struct bounce_zone *bz; 1399166063Scognet int pages; 1400166063Scognet 1401166063Scognet mtx_assert(&bounce_lock, MA_OWNED); 1402166063Scognet bz = dmat->bounce_zone; 1403166063Scognet pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1404166063Scognet if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1405166063Scognet return (map->pagesneeded - (map->pagesreserved + pages)); 1406166063Scognet bz->free_bpages -= pages; 1407166063Scognet bz->reserved_bpages += pages; 1408166063Scognet map->pagesreserved += pages; 1409166063Scognet pages = map->pagesneeded - map->pagesreserved; 1410166063Scognet 1411166063Scognet return (pages); 1412166063Scognet} 1413166063Scognet 1414166063Scognetstatic bus_addr_t 1415166063Scognetadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1416246713Skib bus_addr_t addr, bus_size_t size) 1417166063Scognet{ 1418166063Scognet struct bounce_zone *bz; 1419166063Scognet struct bounce_page *bpage; 1420166063Scognet 1421166063Scognet KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1422170406Scognet KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1423166063Scognet 1424166063Scognet bz = dmat->bounce_zone; 1425166063Scognet if (map->pagesneeded == 0) 1426166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1427166063Scognet map->pagesneeded--; 1428166063Scognet 1429166063Scognet if (map->pagesreserved == 0) 1430166063Scognet panic("add_bounce_page: map doesn't need any pages"); 1431166063Scognet map->pagesreserved--; 1432166063Scognet 1433166063Scognet mtx_lock(&bounce_lock); 1434166063Scognet bpage = STAILQ_FIRST(&bz->bounce_page_list); 1435166063Scognet if (bpage == NULL) 1436166063Scognet panic("add_bounce_page: free page list is empty"); 1437166063Scognet 1438166063Scognet STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1439166063Scognet bz->reserved_bpages--; 1440166063Scognet bz->active_bpages++; 1441166063Scognet mtx_unlock(&bounce_lock); 1442166063Scognet 1443188350Simp if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1444191201Sjhb /* Page offset needs to be preserved. */ 1445282506Shselasky bpage->vaddr |= addr & PAGE_MASK; 1446282506Shselasky bpage->busaddr |= addr & PAGE_MASK; 1447188350Simp } 1448166063Scognet bpage->datavaddr = vaddr; 1449246713Skib bpage->dataaddr = addr; 1450166063Scognet bpage->datacount = size; 1451166063Scognet STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1452166063Scognet return (bpage->busaddr); 1453166063Scognet} 1454166063Scognet 1455166063Scognetstatic void 1456166063Scognetfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1457166063Scognet{ 1458166063Scognet struct bus_dmamap *map; 1459166063Scognet struct bounce_zone *bz; 1460166063Scognet 1461166063Scognet bz = dmat->bounce_zone; 1462166063Scognet bpage->datavaddr = 0; 1463166063Scognet bpage->datacount = 0; 1464191201Sjhb if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1465191201Sjhb /* 1466191201Sjhb * Reset the bounce page to start at offset 0. Other uses 1467191201Sjhb * of this bounce page may need to store a full page of 1468191201Sjhb * data and/or assume it starts on a page boundary. 1469191201Sjhb */ 1470191201Sjhb bpage->vaddr &= ~PAGE_MASK; 1471191201Sjhb bpage->busaddr &= ~PAGE_MASK; 1472191201Sjhb } 1473166063Scognet 1474166063Scognet mtx_lock(&bounce_lock); 1475166063Scognet STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1476166063Scognet bz->free_bpages++; 1477166063Scognet bz->active_bpages--; 1478166063Scognet if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1479166063Scognet if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1480166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1481166063Scognet STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1482166063Scognet map, links); 1483166063Scognet busdma_swi_pending = 1; 1484166063Scognet bz->total_deferred++; 1485166063Scognet swi_sched(vm_ih, 0); 1486166063Scognet } 1487166063Scognet } 1488166063Scognet mtx_unlock(&bounce_lock); 1489166063Scognet} 1490166063Scognet 1491166063Scognetvoid 1492166063Scognetbusdma_swi(void) 1493166063Scognet{ 1494166063Scognet bus_dma_tag_t dmat; 1495166063Scognet struct bus_dmamap *map; 1496166063Scognet 1497166063Scognet mtx_lock(&bounce_lock); 1498166063Scognet while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1499166063Scognet STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1500166063Scognet mtx_unlock(&bounce_lock); 1501166063Scognet dmat = map->dmat; 1502166063Scognet (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1503246713Skib bus_dmamap_load_mem(map->dmat, map, &map->mem, 1504246713Skib map->callback, map->callback_arg, BUS_DMA_WAITOK); 1505166063Scognet (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1506166063Scognet mtx_lock(&bounce_lock); 1507166063Scognet } 1508166063Scognet mtx_unlock(&bounce_lock); 1509166063Scognet} 1510