1139735Simp/*-
2244471Scognet * Copyright (c) 2012 Ian Lepore
3129198Scognet * Copyright (c) 2004 Olivier Houchard
4129198Scognet * Copyright (c) 2002 Peter Grehan
5129198Scognet * Copyright (c) 1997, 1998 Justin T. Gibbs.
6129198Scognet * All rights reserved.
7129198Scognet *
8129198Scognet * Redistribution and use in source and binary forms, with or without
9129198Scognet * modification, are permitted provided that the following conditions
10129198Scognet * are met:
11129198Scognet * 1. Redistributions of source code must retain the above copyright
12129198Scognet *    notice, this list of conditions, and the following disclaimer,
13129198Scognet *    without modification, immediately at the beginning of the file.
14129198Scognet * 2. The name of the author may not be used to endorse or promote products
15129198Scognet *    derived from this software without specific prior written permission.
16129198Scognet *
17129198Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18129198Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19129198Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20129198Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21129198Scognet * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22129198Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23129198Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24129198Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25129198Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26129198Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27129198Scognet * SUCH DAMAGE.
28129198Scognet *
29129198Scognet *   From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred
30129198Scognet */
31129198Scognet
32129198Scognet#include <sys/cdefs.h>
33129198Scognet__FBSDID("$FreeBSD: stable/10/sys/arm/arm/busdma_machdep.c 318977 2017-05-27 08:17:59Z hselasky $");
34129198Scognet
35129198Scognet/*
36244471Scognet * ARM bus dma support routines.
37244471Scognet *
38244471Scognet * XXX Things to investigate / fix some day...
39244471Scognet *  - What is the earliest that this API can be called?  Could there be any
40244471Scognet *    fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
41244471Scognet *  - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the
42244471Scognet *    bus_dmamap_load() function.  This code has historically (and still does)
43244471Scognet *    honor it in bus_dmamem_alloc().  If we got rid of that we could lose some
44244471Scognet *    error checking because some resource management calls would become WAITOK
45244471Scognet *    and thus "cannot fail."
46244471Scognet *  - The decisions made by _bus_dma_can_bounce() should be made once, at tag
47244471Scognet *    creation time, and the result stored in the tag.
48244471Scognet *  - It should be possible to take some shortcuts when mapping a buffer we know
49244471Scognet *    came from the uma(9) allocators based on what we know about such buffers
50244471Scognet *    (aligned, contiguous, etc).
51244471Scognet *  - The allocation of bounce pages could probably be cleaned up, then we could
52244471Scognet *    retire arm_remap_nocache().
53129198Scognet */
54129198Scognet
55129198Scognet#define _ARM32_BUS_DMA_PRIVATE
56129198Scognet#include <sys/param.h>
57129198Scognet#include <sys/systm.h>
58129198Scognet#include <sys/malloc.h>
59129198Scognet#include <sys/bus.h>
60244471Scognet#include <sys/busdma_bufalloc.h>
61129198Scognet#include <sys/interrupt.h>
62129198Scognet#include <sys/lock.h>
63129198Scognet#include <sys/proc.h>
64246713Skib#include <sys/memdesc.h>
65129198Scognet#include <sys/mutex.h>
66140310Scognet#include <sys/ktr.h>
67146597Scognet#include <sys/kernel.h>
68166063Scognet#include <sys/sysctl.h>
69246713Skib#include <sys/uio.h>
70129198Scognet
71244471Scognet#include <vm/uma.h>
72129198Scognet#include <vm/vm.h>
73244471Scognet#include <vm/vm_extern.h>
74244471Scognet#include <vm/vm_kern.h>
75129198Scognet#include <vm/vm_page.h>
76129198Scognet#include <vm/vm_map.h>
77129198Scognet
78129198Scognet#include <machine/atomic.h>
79129198Scognet#include <machine/bus.h>
80129198Scognet#include <machine/cpufunc.h>
81166063Scognet#include <machine/md_var.h>
82129198Scognet
83166063Scognet#define MAX_BPAGES 64
84166063Scognet#define BUS_DMA_COULD_BOUNCE	BUS_DMA_BUS3
85166063Scognet#define BUS_DMA_MIN_ALLOC_COMP	BUS_DMA_BUS4
86166063Scognet
87166063Scognetstruct bounce_zone;
88166063Scognet
89129198Scognetstruct bus_dma_tag {
90129198Scognet	bus_dma_tag_t		parent;
91129198Scognet	bus_size_t		alignment;
92232356Sjhb	bus_addr_t		boundary;
93129198Scognet	bus_addr_t		lowaddr;
94129198Scognet	bus_addr_t		highaddr;
95129198Scognet	bus_dma_filter_t	*filter;
96129198Scognet	void			*filterarg;
97129198Scognet	bus_size_t		maxsize;
98129198Scognet	u_int			nsegments;
99129198Scognet	bus_size_t		maxsegsz;
100129198Scognet	int			flags;
101129198Scognet	int			ref_count;
102129198Scognet	int			map_count;
103129198Scognet	bus_dma_lock_t		*lockfunc;
104129198Scognet	void			*lockfuncarg;
105129198Scognet	/*
106129198Scognet	 * DMA range for this tag.  If the page doesn't fall within
107129198Scognet	 * one of these ranges, an error is returned.  The caller
108129198Scognet	 * may then decide what to do with the transfer.  If the
109129198Scognet	 * range pointer is NULL, it is ignored.
110129198Scognet	 */
111129198Scognet	struct arm32_dma_range	*ranges;
112129198Scognet	int			_nranges;
113166063Scognet	struct bounce_zone *bounce_zone;
114244471Scognet	/*
115244471Scognet	 * Most tags need one or two segments, and can use the local tagsegs
116244471Scognet	 * array.  For tags with a larger limit, we'll allocate a bigger array
117244471Scognet	 * on first use.
118244471Scognet	 */
119244471Scognet	bus_dma_segment_t	*segments;
120244471Scognet	bus_dma_segment_t	tagsegs[2];
121129198Scognet};
122129198Scognet
123166063Scognetstruct bounce_page {
124166063Scognet	vm_offset_t	vaddr;		/* kva of bounce buffer */
125166063Scognet	bus_addr_t	busaddr;	/* Physical address */
126166063Scognet	vm_offset_t	datavaddr;	/* kva of client data */
127246713Skib	bus_addr_t	dataaddr;	/* client physical address */
128166063Scognet	bus_size_t	datacount;	/* client data count */
129166063Scognet	STAILQ_ENTRY(bounce_page) links;
130166063Scognet};
131166063Scognet
132246713Skibstruct sync_list {
133246713Skib	vm_offset_t	vaddr;		/* kva of bounce buffer */
134246713Skib	bus_addr_t	busaddr;	/* Physical address */
135246713Skib	bus_size_t	datacount;	/* client data count */
136246713Skib};
137246713Skib
138166063Scognetint busdma_swi_pending;
139166063Scognet
140166063Scognetstruct bounce_zone {
141166063Scognet	STAILQ_ENTRY(bounce_zone) links;
142166063Scognet	STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
143166063Scognet	int		total_bpages;
144166063Scognet	int		free_bpages;
145166063Scognet	int		reserved_bpages;
146166063Scognet	int		active_bpages;
147166063Scognet	int		total_bounced;
148166063Scognet	int		total_deferred;
149188403Scognet	int		map_count;
150166063Scognet	bus_size_t	alignment;
151166063Scognet	bus_addr_t	lowaddr;
152166063Scognet	char		zoneid[8];
153166063Scognet	char		lowaddrid[20];
154166063Scognet	struct sysctl_ctx_list sysctl_tree;
155166063Scognet	struct sysctl_oid *sysctl_tree_top;
156166063Scognet};
157166063Scognet
158166063Scognetstatic struct mtx bounce_lock;
159166063Scognetstatic int total_bpages;
160166063Scognetstatic int busdma_zonecount;
161166063Scognetstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list;
162166063Scognet
163227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
164166063ScognetSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
165166063Scognet	   "Total bounce pages");
166166063Scognet
167246713Skib#define DMAMAP_COHERENT		0x8
168244471Scognet#define DMAMAP_CACHE_ALIGNED	0x10
169246713Skib
170129198Scognetstruct bus_dmamap {
171166063Scognet	struct bp_list	bpages;
172166063Scognet	int		pagesneeded;
173166063Scognet	int		pagesreserved;
174135644Scognet        bus_dma_tag_t	dmat;
175246713Skib	struct memdesc	mem;
176135644Scognet	int		flags;
177166063Scognet	STAILQ_ENTRY(bus_dmamap) links;
178166063Scognet	bus_dmamap_callback_t *callback;
179166063Scognet	void		      *callback_arg;
180246713Skib	int		       sync_count;
181246713Skib	struct sync_list       *slist;
182129198Scognet};
183129198Scognet
184166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
185166063Scognetstatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
186166063Scognet
187146597Scognetstatic struct mtx busdma_mtx;
188146597Scognet
189146597ScognetMTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
190146597Scognet
191166063Scognetstatic void init_bounce_pages(void *dummy);
192166063Scognetstatic int alloc_bounce_zone(bus_dma_tag_t dmat);
193166063Scognetstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
194166063Scognetstatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
195166063Scognet				int commit);
196166063Scognetstatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
197246713Skib				  vm_offset_t vaddr, bus_addr_t addr,
198246713Skib				  bus_size_t size);
199166063Scognetstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
200166063Scognet
201166063Scognet/* Default tag, as most drivers provide no parent tag. */
202166063Scognetbus_dma_tag_t arm_root_dma_tag;
203166063Scognet
204244473Scognet/*
205244473Scognet * ----------------------------------------------------------------------------
206244473Scognet * Begin block of code useful to transplant to other implementations.
207244473Scognet */
208244471Scognet
209244471Scognetstatic uma_zone_t dmamap_zone;	/* Cache of struct bus_dmamap items */
210244471Scognet
211244471Scognetstatic busdma_bufalloc_t coherent_allocator;	/* Cache of coherent buffers */
212244471Scognetstatic busdma_bufalloc_t standard_allocator;	/* Cache of standard buffers */
213244471Scognet
214166063Scognet/*
215244471Scognet * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
216244471Scognet * It'll need platform-specific changes if this code is copied.
217244471Scognet */
218244471Scognetstatic int
219244471Scognetdmamap_ctor(void *mem, int size, void *arg, int flags)
220244471Scognet{
221244471Scognet	bus_dmamap_t map;
222244471Scognet	bus_dma_tag_t dmat;
223244471Scognet
224244471Scognet	map = (bus_dmamap_t)mem;
225244471Scognet	dmat = (bus_dma_tag_t)arg;
226244471Scognet
227244471Scognet	dmat->map_count++;
228244471Scognet
229244471Scognet	map->dmat = dmat;
230244471Scognet	map->flags = 0;
231244471Scognet	STAILQ_INIT(&map->bpages);
232244471Scognet
233244471Scognet	return (0);
234244471Scognet}
235244471Scognet
236244471Scognet/*
237244471Scognet * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
238244471Scognet * It may need platform-specific changes if this code is copied              .
239244471Scognet */
240244471Scognetstatic void
241244471Scognetdmamap_dtor(void *mem, int size, void *arg)
242244471Scognet{
243244471Scognet	bus_dmamap_t map;
244244471Scognet
245244471Scognet	map = (bus_dmamap_t)mem;
246244471Scognet
247244471Scognet	map->dmat->map_count--;
248244471Scognet}
249244471Scognet
250244471Scognetstatic void
251244471Scognetbusdma_init(void *dummy)
252244471Scognet{
253244471Scognet
254244471Scognet	/* Create a cache of maps for bus_dmamap_create(). */
255244471Scognet	dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
256244471Scognet	    dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
257244471Scognet
258244471Scognet	/* Create a cache of buffers in standard (cacheable) memory. */
259244471Scognet	standard_allocator = busdma_bufalloc_create("buffer",
260244471Scognet	    arm_dcache_align,	/* minimum_alignment */
261244471Scognet	    NULL,		/* uma_alloc func */
262244471Scognet	    NULL,		/* uma_free func */
263244471Scognet	    0);			/* uma_zcreate_flags */
264244471Scognet
265244471Scognet	/*
266244471Scognet	 * Create a cache of buffers in uncacheable memory, to implement the
267244471Scognet	 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag.
268244471Scognet	 */
269244471Scognet	coherent_allocator = busdma_bufalloc_create("coherent",
270244471Scognet	    arm_dcache_align,	/* minimum_alignment */
271244471Scognet	    busdma_bufalloc_alloc_uncacheable,
272244471Scognet	    busdma_bufalloc_free_uncacheable,
273244471Scognet	    0);			/* uma_zcreate_flags */
274244471Scognet}
275244471Scognet
276244471Scognet/*
277244471Scognet * This init historically used SI_SUB_VM, but now the init code requires
278244471Scognet * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by
279244471Scognet * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using
280244471Scognet * SI_SUB_KMEM and SI_ORDER_THIRD.
281244471Scognet */
282244471ScognetSYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL);
283244471Scognet
284244473Scognet/*
285244473Scognet * End block of code useful to transplant to other implementations.
286244473Scognet * ----------------------------------------------------------------------------
287244473Scognet */
288244471Scognet
289244471Scognet/*
290166063Scognet * Return true if a match is made.
291166063Scognet *
292166063Scognet * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
293166063Scognet *
294166063Scognet * If paddr is within the bounds of the dma tag then call the filter callback
295166063Scognet * to check for a match, if there is no filter callback then assume a match.
296166063Scognet */
297166063Scognetstatic int
298166063Scognetrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
299166063Scognet{
300166063Scognet	int retval;
301166063Scognet
302166063Scognet	retval = 0;
303166063Scognet
304166063Scognet	do {
305166063Scognet		if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
306166063Scognet		 || ((paddr & (dmat->alignment - 1)) != 0))
307166063Scognet		 && (dmat->filter == NULL
308166063Scognet		  || (*dmat->filter)(dmat->filterarg, paddr) != 0))
309166063Scognet			retval = 1;
310166063Scognet
311166063Scognet		dmat = dmat->parent;
312166063Scognet	} while (retval == 0 && dmat != NULL);
313166063Scognet	return (retval);
314166063Scognet}
315166063Scognet
316129198Scognet/*
317244471Scognet * This routine checks the exclusion zone constraints from a tag against the
318244471Scognet * physical RAM available on the machine.  If a tag specifies an exclusion zone
319244471Scognet * but there's no RAM in that zone, then we avoid allocating resources to bounce
320244471Scognet * a request, and we can use any memory allocator (as opposed to needing
321244471Scognet * kmem_alloc_contig() just because it can allocate pages in an address range).
322244471Scognet *
323244471Scognet * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the
324244471Scognet * same value on 32-bit architectures) as their lowaddr constraint, and we can't
325244471Scognet * possibly have RAM at an address higher than the highest address we can
326244471Scognet * express, so we take a fast out.
327129198Scognet */
328137758Scognetstatic __inline int
329166063Scognet_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
330166063Scognet{
331166063Scognet	int i;
332244471Scognet
333244471Scognet	if (lowaddr >= BUS_SPACE_MAXADDR)
334244471Scognet		return (0);
335244471Scognet
336166063Scognet	for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
337166063Scognet		if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
338236991Simp		    || (lowaddr < phys_avail[i] &&
339166063Scognet		    highaddr > phys_avail[i]))
340166063Scognet			return (1);
341166063Scognet	}
342166063Scognet	return (0);
343166063Scognet}
344166063Scognet
345129198Scognetstatic __inline struct arm32_dma_range *
346129198Scognet_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
347129198Scognet    bus_addr_t curaddr)
348129198Scognet{
349129198Scognet	struct arm32_dma_range *dr;
350129198Scognet	int i;
351129198Scognet
352129198Scognet	for (i = 0, dr = ranges; i < nranges; i++, dr++) {
353129198Scognet		if (curaddr >= dr->dr_sysbase &&
354129198Scognet		    round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
355129198Scognet			return (dr);
356129198Scognet	}
357129198Scognet
358129198Scognet	return (NULL);
359129198Scognet}
360129198Scognet/*
361129198Scognet * Convenience function for manipulating driver locks from busdma (during
362129198Scognet * busdma_swi, for example).  Drivers that don't provide their own locks
363129198Scognet * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
364129198Scognet * non-mutex locking scheme don't have to use this at all.
365129198Scognet */
366129198Scognetvoid
367129198Scognetbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
368129198Scognet{
369129198Scognet	struct mtx *dmtx;
370129198Scognet
371129198Scognet	dmtx = (struct mtx *)arg;
372129198Scognet	switch (op) {
373129198Scognet	case BUS_DMA_LOCK:
374129198Scognet		mtx_lock(dmtx);
375129198Scognet		break;
376129198Scognet	case BUS_DMA_UNLOCK:
377129198Scognet		mtx_unlock(dmtx);
378129198Scognet		break;
379129198Scognet	default:
380129198Scognet		panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
381129198Scognet	}
382129198Scognet}
383129198Scognet
384129198Scognet/*
385129198Scognet * dflt_lock should never get called.  It gets put into the dma tag when
386129198Scognet * lockfunc == NULL, which is only valid if the maps that are associated
387129198Scognet * with the tag are meant to never be defered.
388129198Scognet * XXX Should have a way to identify which driver is responsible here.
389129198Scognet */
390129198Scognetstatic void
391129198Scognetdflt_lock(void *arg, bus_dma_lock_op_t op)
392129198Scognet{
393129198Scognet#ifdef INVARIANTS
394129198Scognet	panic("driver error: busdma dflt_lock called");
395129198Scognet#else
396129198Scognet	printf("DRIVER_ERROR: busdma dflt_lock called\n");
397129198Scognet#endif
398129198Scognet}
399129198Scognet
400129198Scognet/*
401129198Scognet * Allocate a device specific dma_tag.
402129198Scognet */
403135644Scognet#define SEG_NB 1024
404135644Scognet
405129198Scognetint
406129198Scognetbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
407232356Sjhb		   bus_addr_t boundary, bus_addr_t lowaddr,
408129198Scognet		   bus_addr_t highaddr, bus_dma_filter_t *filter,
409129198Scognet		   void *filterarg, bus_size_t maxsize, int nsegments,
410129198Scognet		   bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
411129198Scognet		   void *lockfuncarg, bus_dma_tag_t *dmat)
412129198Scognet{
413129198Scognet	bus_dma_tag_t newtag;
414129198Scognet	int error = 0;
415129198Scognet	/* Return a NULL tag on failure */
416129198Scognet	*dmat = NULL;
417166063Scognet	if (!parent)
418166063Scognet		parent = arm_root_dma_tag;
419129198Scognet
420129198Scognet	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
421140313Scognet	if (newtag == NULL) {
422143294Smux		CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
423143284Smux		    __func__, newtag, 0, error);
424129198Scognet		return (ENOMEM);
425140313Scognet	}
426129198Scognet
427129198Scognet	newtag->parent = parent;
428244471Scognet	newtag->alignment = alignment ? alignment : 1;
429129198Scognet	newtag->boundary = boundary;
430129198Scognet	newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
431129198Scognet	newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
432129198Scognet	newtag->filter = filter;
433129198Scognet	newtag->filterarg = filterarg;
434129198Scognet        newtag->maxsize = maxsize;
435129198Scognet        newtag->nsegments = nsegments;
436129198Scognet	newtag->maxsegsz = maxsegsz;
437129198Scognet	newtag->flags = flags;
438129198Scognet	newtag->ref_count = 1; /* Count ourself */
439129198Scognet	newtag->map_count = 0;
440129198Scognet	newtag->ranges = bus_dma_get_range();
441135644Scognet	newtag->_nranges = bus_dma_get_range_nb();
442129198Scognet	if (lockfunc != NULL) {
443129198Scognet		newtag->lockfunc = lockfunc;
444129198Scognet		newtag->lockfuncarg = lockfuncarg;
445129198Scognet	} else {
446129198Scognet		newtag->lockfunc = dflt_lock;
447129198Scognet		newtag->lockfuncarg = NULL;
448129198Scognet	}
449244471Scognet	/*
450244471Scognet	 * If all the segments we need fit into the local tagsegs array, set the
451244471Scognet	 * pointer now.  Otherwise NULL the pointer and an array of segments
452244471Scognet	 * will be allocated later, on first use.  We don't pre-allocate now
453244471Scognet	 * because some tags exist just to pass contraints to children in the
454244471Scognet	 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we
455244471Scognet	 * sure don't want to try to allocate an array for that.
456244471Scognet	 */
457244471Scognet	if (newtag->nsegments <= nitems(newtag->tagsegs))
458244471Scognet		newtag->segments = newtag->tagsegs;
459244471Scognet	else
460244471Scognet		newtag->segments = NULL;
461244471Scognet	/*
462129198Scognet	 * Take into account any restrictions imposed by our parent tag
463129198Scognet	 */
464129198Scognet        if (parent != NULL) {
465232356Sjhb                newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
466232356Sjhb                newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
467134934Sscottl		if (newtag->boundary == 0)
468134934Sscottl			newtag->boundary = parent->boundary;
469134934Sscottl		else if (parent->boundary != 0)
470232356Sjhb                	newtag->boundary = MIN(parent->boundary,
471134934Sscottl					       newtag->boundary);
472166063Scognet		if ((newtag->filter != NULL) ||
473166063Scognet		    ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
474166063Scognet			newtag->flags |= BUS_DMA_COULD_BOUNCE;
475129198Scognet                if (newtag->filter == NULL) {
476129198Scognet                        /*
477129198Scognet                         * Short circuit looking at our parent directly
478129198Scognet                         * since we have encapsulated all of its information
479129198Scognet                         */
480129198Scognet                        newtag->filter = parent->filter;
481129198Scognet                        newtag->filterarg = parent->filterarg;
482129198Scognet                        newtag->parent = parent->parent;
483129198Scognet		}
484129198Scognet		if (newtag->parent != NULL)
485129198Scognet			atomic_add_int(&parent->ref_count, 1);
486129198Scognet	}
487166063Scognet	if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
488166063Scognet	 || newtag->alignment > 1)
489166063Scognet		newtag->flags |= BUS_DMA_COULD_BOUNCE;
490129198Scognet
491166063Scognet	if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
492166063Scognet	    (flags & BUS_DMA_ALLOCNOW) != 0) {
493166063Scognet		struct bounce_zone *bz;
494166063Scognet
495166063Scognet		/* Must bounce */
496166063Scognet
497166063Scognet		if ((error = alloc_bounce_zone(newtag)) != 0) {
498166063Scognet			free(newtag, M_DEVBUF);
499166063Scognet			return (error);
500166063Scognet		}
501166063Scognet		bz = newtag->bounce_zone;
502166063Scognet
503166063Scognet		if (ptoa(bz->total_bpages) < maxsize) {
504166063Scognet			int pages;
505166063Scognet
506166063Scognet			pages = atop(maxsize) - bz->total_bpages;
507166063Scognet
508166063Scognet			/* Add pages to our bounce pool */
509166063Scognet			if (alloc_bounce_pages(newtag, pages) < pages)
510166063Scognet				error = ENOMEM;
511166063Scognet		}
512166063Scognet		/* Performed initial allocation */
513166063Scognet		newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
514170502Scognet	} else
515170502Scognet		newtag->bounce_zone = NULL;
516166063Scognet	if (error != 0)
517166063Scognet		free(newtag, M_DEVBUF);
518166063Scognet	else
519166063Scognet		*dmat = newtag;
520143294Smux	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
521143284Smux	    __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
522140313Scognet
523129198Scognet	return (error);
524129198Scognet}
525129198Scognet
526129198Scognetint
527129198Scognetbus_dma_tag_destroy(bus_dma_tag_t dmat)
528129198Scognet{
529140680Scognet#ifdef KTR
530140313Scognet	bus_dma_tag_t dmat_copy = dmat;
531140680Scognet#endif
532140313Scognet
533129198Scognet	if (dmat != NULL) {
534244471Scognet
535129198Scognet                if (dmat->map_count != 0)
536129198Scognet                        return (EBUSY);
537129198Scognet
538129198Scognet                while (dmat != NULL) {
539129198Scognet                        bus_dma_tag_t parent;
540129198Scognet
541129198Scognet                        parent = dmat->parent;
542129198Scognet                        atomic_subtract_int(&dmat->ref_count, 1);
543129198Scognet                        if (dmat->ref_count == 0) {
544244471Scognet				if (dmat->segments != NULL &&
545244471Scognet				    dmat->segments != dmat->tagsegs)
546240177Sjhb					free(dmat->segments, M_DEVBUF);
547129198Scognet                                free(dmat, M_DEVBUF);
548129198Scognet                                /*
549129198Scognet                                 * Last reference count, so
550129198Scognet                                 * release our reference
551129198Scognet                                 * count on our parent.
552129198Scognet                                 */
553129198Scognet                                dmat = parent;
554129198Scognet                        } else
555129198Scognet                                dmat = NULL;
556129198Scognet                }
557129198Scognet        }
558143294Smux	CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
559140313Scognet
560129198Scognet        return (0);
561129198Scognet}
562129198Scognet
563166063Scognet#include <sys/kdb.h>
564129198Scognet/*
565129198Scognet * Allocate a handle for mapping from kva/uva/physical
566129198Scognet * address space into bus device space.
567129198Scognet */
568129198Scognetint
569129198Scognetbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
570129198Scognet{
571246713Skib	struct sync_list *slist;
572244471Scognet	bus_dmamap_t map;
573140313Scognet	int error = 0;
574129198Scognet
575246713Skib	slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
576246713Skib	if (slist == NULL)
577246713Skib		return (ENOMEM);
578246713Skib
579244575Scognet	map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT);
580244471Scognet	*mapp = map;
581246713Skib	if (map == NULL) {
582246713Skib		free(slist, M_DEVBUF);
583244575Scognet		return (ENOMEM);
584246713Skib	}
585240177Sjhb
586244471Scognet	/*
587244471Scognet	 * If the tag's segments haven't been allocated yet we need to do it
588244471Scognet	 * now, because we can't sleep for resources at map load time.
589244471Scognet	 */
590244575Scognet	if (dmat->segments == NULL) {
591244471Scognet		dmat->segments = malloc(dmat->nsegments *
592244575Scognet		    sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT);
593244575Scognet		if (dmat->segments == NULL) {
594246713Skib			free(slist, M_DEVBUF);
595244575Scognet			uma_zfree(dmamap_zone, map);
596244575Scognet			*mapp = NULL;
597244575Scognet			return (ENOMEM);
598244575Scognet		}
599244575Scognet	}
600129198Scognet
601166063Scognet	/*
602166063Scognet	 * Bouncing might be required if the driver asks for an active
603166063Scognet	 * exclusion region, a data alignment that is stricter than 1, and/or
604166063Scognet	 * an active address boundary.
605166063Scognet	 */
606166063Scognet	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
607166063Scognet
608166063Scognet		/* Must bounce */
609166063Scognet		struct bounce_zone *bz;
610166063Scognet		int maxpages;
611166063Scognet
612166063Scognet		if (dmat->bounce_zone == NULL) {
613166063Scognet			if ((error = alloc_bounce_zone(dmat)) != 0) {
614246713Skib				free(slist, M_DEVBUF);
615244471Scognet				uma_zfree(dmamap_zone, map);
616166063Scognet				*mapp = NULL;
617166063Scognet				return (error);
618166063Scognet			}
619166063Scognet		}
620166063Scognet		bz = dmat->bounce_zone;
621166063Scognet
622166063Scognet		/* Initialize the new map */
623166063Scognet		STAILQ_INIT(&((*mapp)->bpages));
624166063Scognet
625166063Scognet		/*
626166063Scognet		 * Attempt to add pages to our pool on a per-instance
627166063Scognet		 * basis up to a sane limit.
628166063Scognet		 */
629166063Scognet		maxpages = MAX_BPAGES;
630166063Scognet		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
631188403Scognet		 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
632166063Scognet			int pages;
633166063Scognet
634166063Scognet			pages = MAX(atop(dmat->maxsize), 1);
635166063Scognet			pages = MIN(maxpages - bz->total_bpages, pages);
636166063Scognet			pages = MAX(pages, 1);
637166063Scognet			if (alloc_bounce_pages(dmat, pages) < pages)
638166063Scognet				error = ENOMEM;
639166063Scognet
640166063Scognet			if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
641166063Scognet				if (error == 0)
642166063Scognet					dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
643166063Scognet			} else {
644166063Scognet				error = 0;
645166063Scognet			}
646166063Scognet		}
647188403Scognet		bz->map_count++;
648166063Scognet	}
649246713Skib	map->sync_count = 0;
650246713Skib	map->slist = slist;
651143294Smux	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
652143284Smux	    __func__, dmat, dmat->flags, error);
653140313Scognet
654129198Scognet	return (0);
655129198Scognet}
656129198Scognet
657129198Scognet/*
658129198Scognet * Destroy a handle for mapping from kva/uva/physical
659129198Scognet * address space into bus device space.
660129198Scognet */
661129198Scognetint
662129198Scognetbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
663129198Scognet{
664135644Scognet
665246713Skib	if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
666166063Scognet		CTR3(KTR_BUSDMA, "%s: tag %p error %d",
667166063Scognet		    __func__, dmat, EBUSY);
668166063Scognet		return (EBUSY);
669166063Scognet	}
670246713Skib	free(map->slist, M_DEVBUF);
671244471Scognet	uma_zfree(dmamap_zone, map);
672188403Scognet	if (dmat->bounce_zone)
673188403Scognet		dmat->bounce_zone->map_count--;
674143294Smux	CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
675129198Scognet        return (0);
676129198Scognet}
677129198Scognet
678129198Scognet/*
679244471Scognet * Allocate a piece of memory that can be efficiently mapped into bus device
680244471Scognet * space based on the constraints listed in the dma tag.  Returns a pointer to
681244471Scognet * the allocated memory, and a pointer to an associated bus_dmamap.
682129198Scognet */
683129198Scognetint
684244471Scognetbus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags,
685129198Scognet                 bus_dmamap_t *mapp)
686129198Scognet{
687246713Skib	struct sync_list *slist;
688244471Scognet	void * vaddr;
689244471Scognet	struct busdma_bufzone *bufzone;
690244471Scognet	busdma_bufalloc_t ba;
691244471Scognet	bus_dmamap_t map;
692129198Scognet	int mflags;
693244471Scognet	vm_memattr_t memattr;
694129198Scognet
695129198Scognet	if (flags & BUS_DMA_NOWAIT)
696129198Scognet		mflags = M_NOWAIT;
697129198Scognet	else
698129198Scognet		mflags = M_WAITOK;
699244471Scognet	/*
700244471Scognet	 * If the tag's segments haven't been allocated yet we need to do it
701244471Scognet	 * now, because we can't sleep for resources at map load time.
702244471Scognet	 */
703244471Scognet	if (dmat->segments == NULL)
704244471Scognet		dmat->segments = malloc(dmat->nsegments *
705244471Scognet		   sizeof(*dmat->segments), M_DEVBUF, mflags);
706244471Scognet
707246713Skib	slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
708246713Skib	if (slist == NULL)
709246713Skib		return (ENOMEM);
710244471Scognet	map = uma_zalloc_arg(dmamap_zone, dmat, mflags);
711246713Skib	if (map == NULL) {
712246713Skib		free(slist, M_DEVBUF);
713244471Scognet		return (ENOMEM);
714246713Skib	}
715244471Scognet	if (flags & BUS_DMA_COHERENT) {
716244471Scognet		memattr = VM_MEMATTR_UNCACHEABLE;
717244471Scognet		ba = coherent_allocator;
718244471Scognet		map->flags |= DMAMAP_COHERENT;
719244471Scognet	} else {
720244471Scognet		memattr = VM_MEMATTR_DEFAULT;
721244471Scognet		ba = standard_allocator;
722240177Sjhb	}
723244471Scognet	/* All buffers we allocate are cache-aligned. */
724244471Scognet	map->flags |= DMAMAP_CACHE_ALIGNED;
725244471Scognet
726129198Scognet	if (flags & BUS_DMA_ZERO)
727129198Scognet		mflags |= M_ZERO;
728129198Scognet
729244471Scognet	/*
730244471Scognet	 * Try to find a bufzone in the allocator that holds a cache of buffers
731244471Scognet	 * of the right size for this request.  If the buffer is too big to be
732244471Scognet	 * held in the allocator cache, this returns NULL.
733244471Scognet	 */
734244471Scognet	bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
735244471Scognet
736244471Scognet	/*
737244471Scognet	 * Allocate the buffer from the uma(9) allocator if...
738244471Scognet	 *  - It's small enough to be in the allocator (bufzone not NULL).
739244471Scognet	 *  - The alignment constraint isn't larger than the allocation size
740244471Scognet	 *    (the allocator aligns buffers to their size boundaries).
741244471Scognet	 *  - There's no need to handle lowaddr/highaddr exclusion zones.
742244471Scognet	 * else allocate non-contiguous pages if...
743244471Scognet	 *  - The page count that could get allocated doesn't exceed nsegments.
744244471Scognet	 *  - The alignment constraint isn't larger than a page boundary.
745244471Scognet	 *  - There are no boundary-crossing constraints.
746244471Scognet	 * else allocate a block of contiguous pages because one or more of the
747244471Scognet	 * constraints is something that only the contig allocator can fulfill.
748244471Scognet	 */
749244471Scognet	if (bufzone != NULL && dmat->alignment <= bufzone->size &&
750244471Scognet	    !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
751244471Scognet		vaddr = uma_zalloc(bufzone->umazone, mflags);
752318977Shselasky	} else if (dmat->nsegments >=
753318977Shselasky	    howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) &&
754318977Shselasky	    dmat->alignment <= PAGE_SIZE &&
755318977Shselasky	    (dmat->boundary % PAGE_SIZE) == 0) {
756254025Sjeff		vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
757244471Scognet		    mflags, 0, dmat->lowaddr, memattr);
758244471Scognet	} else {
759254025Sjeff		vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
760244471Scognet		    mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
761244471Scognet		    memattr);
762135644Scognet	}
763244471Scognet	if (vaddr == NULL) {
764246713Skib		free(slist, M_DEVBUF);
765244471Scognet		uma_zfree(dmamap_zone, map);
766244471Scognet		map = NULL;
767246713Skib	} else {
768246713Skib		map->slist = slist;
769246713Skib		map->sync_count = 0;
770129198Scognet	}
771244471Scognet	*vaddrp = vaddr;
772244471Scognet	*mapp = map;
773244471Scognet
774244471Scognet	return (vaddr == NULL ? ENOMEM : 0);
775129198Scognet}
776129198Scognet
777129198Scognet/*
778244471Scognet * Free a piece of memory that was allocated via bus_dmamem_alloc, along with
779244471Scognet * its associated map.
780129198Scognet */
781129198Scognetvoid
782129198Scognetbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
783129198Scognet{
784244471Scognet	struct busdma_bufzone *bufzone;
785244471Scognet	busdma_bufalloc_t ba;
786244471Scognet
787244471Scognet	if (map->flags & DMAMAP_COHERENT)
788244471Scognet		ba = coherent_allocator;
789246713Skib	else
790244471Scognet		ba = standard_allocator;
791244471Scognet
792246713Skib	free(map->slist, M_DEVBUF);
793294671Sian	uma_zfree(dmamap_zone, map);
794244471Scognet
795244471Scognet	bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
796244471Scognet
797244471Scognet	if (bufzone != NULL && dmat->alignment <= bufzone->size &&
798166063Scognet	    !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
799244471Scognet		uma_zfree(bufzone->umazone, vaddr);
800244471Scognet	else
801254025Sjeff		kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
802129198Scognet}
803129198Scognet
804246713Skibstatic void
805246713Skib_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
806246713Skib    bus_size_t buflen, int flags)
807246713Skib{
808246713Skib	bus_addr_t curaddr;
809246713Skib	bus_size_t sgsize;
810246713Skib
811259337Sian	if (map->pagesneeded == 0) {
812246713Skib		CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
813246713Skib		    dmat->lowaddr, dmat->boundary, dmat->alignment);
814246713Skib		CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
815246713Skib		    map, map->pagesneeded);
816246713Skib		/*
817246713Skib		 * Count the number of bounce pages
818246713Skib		 * needed in order to complete this transfer
819246713Skib		 */
820246713Skib		curaddr = buf;
821246713Skib		while (buflen != 0) {
822246713Skib			sgsize = MIN(buflen, dmat->maxsegsz);
823246713Skib			if (run_filter(dmat, curaddr) != 0) {
824246713Skib				sgsize = MIN(sgsize, PAGE_SIZE);
825246713Skib				map->pagesneeded++;
826246713Skib			}
827246713Skib			curaddr += sgsize;
828246713Skib			buflen -= sgsize;
829246713Skib		}
830246713Skib		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
831246713Skib	}
832246713Skib}
833246713Skib
834246713Skibstatic void
835191011Skib_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
836191011Skib    void *buf, bus_size_t buflen, int flags)
837166063Scognet{
838166063Scognet	vm_offset_t vaddr;
839166063Scognet	vm_offset_t vendaddr;
840166063Scognet	bus_addr_t paddr;
841166063Scognet
842259337Sian	if (map->pagesneeded == 0) {
843185494Sstas		CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
844185494Sstas		    dmat->lowaddr, dmat->boundary, dmat->alignment);
845170406Scognet		CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
846170406Scognet		    map, map->pagesneeded);
847166063Scognet		/*
848166063Scognet		 * Count the number of bounce pages
849166063Scognet		 * needed in order to complete this transfer
850166063Scognet		 */
851166063Scognet		vaddr = trunc_page((vm_offset_t)buf);
852166063Scognet		vendaddr = (vm_offset_t)buf + buflen;
853166063Scognet
854166063Scognet		while (vaddr < vendaddr) {
855246713Skib			if (__predict_true(pmap == kernel_pmap))
856191438Sjhb				paddr = pmap_kextract(vaddr);
857191438Sjhb			else
858191011Skib				paddr = pmap_extract(pmap, vaddr);
859246713Skib			if (run_filter(dmat, paddr) != 0)
860166063Scognet				map->pagesneeded++;
861166063Scognet			vaddr += PAGE_SIZE;
862166063Scognet		}
863166063Scognet		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
864166063Scognet	}
865246713Skib}
866166063Scognet
867246713Skibstatic int
868246713Skib_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
869246713Skib{
870246713Skib
871166063Scognet	/* Reserve Necessary Bounce Pages */
872246713Skib	mtx_lock(&bounce_lock);
873246713Skib	if (flags & BUS_DMA_NOWAIT) {
874246713Skib		if (reserve_bounce_pages(dmat, map, 0) != 0) {
875246713Skib			mtx_unlock(&bounce_lock);
876246713Skib			return (ENOMEM);
877166063Scognet		}
878246713Skib	} else {
879246713Skib		if (reserve_bounce_pages(dmat, map, 1) != 0) {
880246713Skib			/* Queue us for resources */
881246713Skib			STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
882246713Skib			mtx_unlock(&bounce_lock);
883246713Skib			return (EINPROGRESS);
884246713Skib		}
885166063Scognet	}
886246713Skib	mtx_unlock(&bounce_lock);
887166063Scognet
888166063Scognet	return (0);
889166063Scognet}
890166063Scognet
891129198Scognet/*
892246713Skib * Add a single contiguous physical range to the segment list.
893246713Skib */
894246713Skibstatic int
895246713Skib_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
896246713Skib    bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
897246713Skib{
898246713Skib	bus_addr_t baddr, bmask;
899246713Skib	int seg;
900246713Skib
901246713Skib	/*
902246713Skib	 * Make sure we don't cross any boundaries.
903246713Skib	 */
904246713Skib	bmask = ~(dmat->boundary - 1);
905246713Skib	if (dmat->boundary > 0) {
906246713Skib		baddr = (curaddr + dmat->boundary) & bmask;
907246713Skib		if (sgsize > (baddr - curaddr))
908246713Skib			sgsize = (baddr - curaddr);
909246713Skib	}
910246713Skib	if (dmat->ranges) {
911246713Skib		struct arm32_dma_range *dr;
912246713Skib
913246713Skib		dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
914246713Skib		    curaddr);
915246713Skib		if (dr == NULL)
916246881Sian			return (0);
917246713Skib		/*
918246713Skib		 * In a valid DMA range.  Translate the physical
919246713Skib		 * memory address to an address in the DMA window.
920246713Skib		 */
921246713Skib		curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
922246713Skib
923246713Skib	}
924246713Skib
925246713Skib	seg = *segp;
926246713Skib	/*
927246713Skib	 * Insert chunk into a segment, coalescing with
928246713Skib	 * the previous segment if possible.
929246713Skib	 */
930246713Skib	if (seg >= 0 &&
931246713Skib	    curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
932246713Skib	    (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
933246713Skib	    (dmat->boundary == 0 ||
934246713Skib	     (segs[seg].ds_addr & bmask) == (curaddr & bmask))) {
935246713Skib		segs[seg].ds_len += sgsize;
936246713Skib	} else {
937246713Skib		if (++seg >= dmat->nsegments)
938246881Sian			return (0);
939246713Skib		segs[seg].ds_addr = curaddr;
940246713Skib		segs[seg].ds_len = sgsize;
941246713Skib	}
942246713Skib	*segp = seg;
943246881Sian	return (sgsize);
944246713Skib}
945246713Skib
946246713Skib/*
947246713Skib * Utility function to load a physical buffer.  segp contains
948246713Skib * the starting segment on entrace, and the ending segment on exit.
949246713Skib */
950246713Skibint
951246713Skib_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
952246713Skib    bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
953246713Skib{
954246713Skib	bus_size_t sgsize;
955246713Skib	bus_addr_t curaddr;
956246713Skib	int error;
957246713Skib
958246713Skib	if (segs == NULL)
959246713Skib		segs = dmat->segments;
960246713Skib
961246713Skib	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
962246713Skib		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
963246713Skib		if (map->pagesneeded != 0) {
964246713Skib			error = _bus_dmamap_reserve_pages(dmat, map, flags);
965246713Skib			if (error)
966246713Skib				return (error);
967246713Skib		}
968246713Skib	}
969246713Skib
970246713Skib	while (buflen > 0) {
971246713Skib		curaddr = buf;
972246713Skib		sgsize = MIN(buflen, dmat->maxsegsz);
973246713Skib		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
974246713Skib		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
975246713Skib			sgsize = MIN(sgsize, PAGE_SIZE);
976246713Skib			curaddr = add_bounce_page(dmat, map, 0, curaddr,
977246713Skib			    sgsize);
978246713Skib		}
979246713Skib		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
980246713Skib		    segp);
981246713Skib		if (sgsize == 0)
982246713Skib			break;
983246713Skib		buf += sgsize;
984246713Skib		buflen -= sgsize;
985246713Skib	}
986246713Skib
987246713Skib	/*
988246713Skib	 * Did we fit?
989246713Skib	 */
990246713Skib	if (buflen != 0) {
991246713Skib		_bus_dmamap_unload(dmat, map);
992246713Skib		return (EFBIG); /* XXX better return value here? */
993246713Skib	}
994246713Skib	return (0);
995246713Skib}
996259510Skib
997259510Skibint
998259510Skib_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
999259510Skib    struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
1000259510Skib    bus_dma_segment_t *segs, int *segp)
1001259510Skib{
1002259510Skib
1003259510Skib	return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
1004259510Skib	    segs, segp));
1005259510Skib}
1006259510Skib
1007246713Skib/*
1008246713Skib * Utility function to load a linear buffer.  segp contains
1009129198Scognet * the starting segment on entrance, and the ending segment on exit.
1010129198Scognet */
1011246713Skibint
1012246713Skib_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
1013246713Skib    bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
1014246713Skib    int *segp)
1015129198Scognet{
1016129198Scognet	bus_size_t sgsize;
1017246713Skib	bus_addr_t curaddr;
1018246713Skib	struct sync_list *sl;
1019129198Scognet	vm_offset_t vaddr = (vm_offset_t)buf;
1020129198Scognet	int error = 0;
1021129198Scognet
1022246713Skib	if (segs == NULL)
1023246713Skib		segs = dmat->segments;
1024246713Skib	if ((flags & BUS_DMA_LOAD_MBUF) != 0)
1025246713Skib		map->flags |= DMAMAP_CACHE_ALIGNED;
1026129198Scognet
1027166063Scognet	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
1028246713Skib		_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
1029246713Skib		if (map->pagesneeded != 0) {
1030246713Skib			error = _bus_dmamap_reserve_pages(dmat, map, flags);
1031246713Skib			if (error)
1032246713Skib				return (error);
1033246713Skib		}
1034166063Scognet	}
1035140313Scognet	CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
1036140313Scognet	    "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
1037140313Scognet
1038246713Skib	while (buflen > 0) {
1039129198Scognet		/*
1040129198Scognet		 * Get the physical address for this segment.
1041129198Scognet		 */
1042246713Skib		if (__predict_true(pmap == kernel_pmap)) {
1043246158Skib			curaddr = pmap_kextract(vaddr);
1044129198Scognet		} else {
1045129198Scognet			curaddr = pmap_extract(pmap, vaddr);
1046135644Scognet			map->flags &= ~DMAMAP_COHERENT;
1047129198Scognet		}
1048129198Scognet
1049129198Scognet		/*
1050129198Scognet		 * Compute the segment size, and adjust counts.
1051129198Scognet		 */
1052129198Scognet		sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
1053170086Syongari		if (sgsize > dmat->maxsegsz)
1054170086Syongari			sgsize = dmat->maxsegsz;
1055129198Scognet		if (buflen < sgsize)
1056129198Scognet			sgsize = buflen;
1057129198Scognet
1058166063Scognet		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
1059246713Skib		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
1060246713Skib			curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
1061246713Skib			    sgsize);
1062137760Scognet		} else {
1063246713Skib			sl = &map->slist[map->sync_count - 1];
1064246713Skib			if (map->sync_count == 0 ||
1065246713Skib			    vaddr != sl->vaddr + sl->datacount) {
1066246713Skib				if (++map->sync_count > dmat->nsegments)
1067246713Skib					goto cleanup;
1068246713Skib				sl++;
1069246713Skib				sl->vaddr = vaddr;
1070246713Skib				sl->datacount = sgsize;
1071246713Skib				sl->busaddr = curaddr;
1072246713Skib			} else
1073246713Skib				sl->datacount += sgsize;
1074129198Scognet		}
1075246713Skib		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1076246713Skib		    segp);
1077246713Skib		if (sgsize == 0)
1078135644Scognet			break;
1079129198Scognet		vaddr += sgsize;
1080129198Scognet		buflen -= sgsize;
1081129198Scognet	}
1082129198Scognet
1083246713Skibcleanup:
1084129198Scognet	/*
1085129198Scognet	 * Did we fit?
1086129198Scognet	 */
1087246713Skib	if (buflen != 0) {
1088246713Skib		_bus_dmamap_unload(dmat, map);
1089246713Skib		return (EFBIG); /* XXX better return value here? */
1090246713Skib	}
1091246713Skib	return (0);
1092129198Scognet}
1093129198Scognet
1094246713Skibvoid
1095246713Skib__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
1096246713Skib		    struct memdesc *mem, bus_dmamap_callback_t *callback,
1097246713Skib		    void *callback_arg)
1098140682Scognet{
1099140682Scognet
1100143671Sjmg	KASSERT(dmat != NULL, ("dmatag is NULL"));
1101143671Sjmg	KASSERT(map != NULL, ("dmamap is NULL"));
1102246713Skib	map->mem = *mem;
1103166063Scognet	map->callback = callback;
1104166063Scognet	map->callback_arg = callback_arg;
1105140682Scognet}
1106140682Scognet
1107246713Skibbus_dma_segment_t *
1108246713Skib_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
1109246713Skib		     bus_dma_segment_t *segs, int nsegs, int error)
1110129198Scognet{
1111129198Scognet
1112246713Skib	if (segs == NULL)
1113246713Skib		segs = dmat->segments;
1114246713Skib	return (segs);
1115129198Scognet}
1116129198Scognet
1117129198Scognet/*
1118135644Scognet * Release the mapping held by map.
1119129198Scognet */
1120129198Scognetvoid
1121143655Sjmg_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
1122150893Scognet{
1123166063Scognet	struct bounce_page *bpage;
1124166063Scognet
1125166063Scognet	while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1126166063Scognet		STAILQ_REMOVE_HEAD(&map->bpages, links);
1127166063Scognet		free_bounce_page(dmat, bpage);
1128166063Scognet	}
1129246713Skib	map->sync_count = 0;
1130129198Scognet	return;
1131129198Scognet}
1132129198Scognet
1133169761Scognetstatic void
1134246713Skibbus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op,
1135246713Skib    int bufaligned)
1136135644Scognet{
1137166063Scognet	char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
1138234561Smarius	register_t s;
1139236991Simp	int partial;
1140135644Scognet
1141171890Scognet	if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) {
1142246713Skib		cpu_dcache_wb_range(buf, len);
1143246713Skib		cpu_l2cache_wb_range(buf, len);
1144171623Scognet	}
1145244471Scognet
1146244471Scognet	/*
1147244471Scognet	 * If the caller promises the buffer is properly aligned to a cache line
1148244471Scognet	 * (even if the call parms make it look like it isn't) we can avoid
1149244471Scognet	 * attempting to preserve the non-DMA part of the cache line in the
1150244471Scognet	 * POSTREAD case, but we MUST still do a writeback in the PREREAD case.
1151244471Scognet	 *
1152244471Scognet	 * This covers the case of mbufs, where we know how they're aligned and
1153244471Scognet	 * know the CPU doesn't touch the header in front of the DMA data area
1154244471Scognet	 * during the IO, but it may have touched it right before invoking the
1155244471Scognet	 * sync, so a PREREAD writeback is required.
1156244471Scognet	 *
1157244471Scognet	 * It also handles buffers we created in bus_dmamem_alloc(), which are
1158244471Scognet	 * always aligned and padded to cache line size even if the IO length
1159244471Scognet	 * isn't a multiple of cache line size.  In this case the PREREAD
1160244471Scognet	 * writeback probably isn't required, but it's harmless.
1161244471Scognet	 */
1162234561Smarius	partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask;
1163244471Scognet
1164171623Scognet	if (op & BUS_DMASYNC_PREREAD) {
1165234561Smarius		if (!(op & BUS_DMASYNC_PREWRITE) && !partial) {
1166246713Skib			cpu_dcache_inv_range(buf, len);
1167246713Skib			cpu_l2cache_inv_range(buf, len);
1168171890Scognet		} else {
1169246713Skib		    	cpu_dcache_wbinv_range(buf, len);
1170246713Skib	    		cpu_l2cache_wbinv_range(buf, len);
1171171890Scognet		}
1172171623Scognet	}
1173166063Scognet	if (op & BUS_DMASYNC_POSTREAD) {
1174244471Scognet		if (partial && !bufaligned) {
1175234561Smarius			s = intr_disable();
1176246713Skib			if (buf & arm_dcache_align_mask)
1177246713Skib				memcpy(_tmp_cl, (void *)(buf &
1178234561Smarius				    ~arm_dcache_align_mask),
1179246713Skib				    buf & arm_dcache_align_mask);
1180246713Skib			if ((buf + len) & arm_dcache_align_mask)
1181236991Simp				memcpy(_tmp_clend,
1182246713Skib				    (void *)(buf + len),
1183246713Skib				    arm_dcache_align -
1184246713Skib				    ((buf + len) & arm_dcache_align_mask));
1185171623Scognet		}
1186246713Skib		cpu_dcache_inv_range(buf, len);
1187246713Skib		cpu_l2cache_inv_range(buf, len);
1188244471Scognet		if (partial && !bufaligned) {
1189246713Skib			if (buf & arm_dcache_align_mask)
1190246713Skib				memcpy((void *)(buf &
1191236991Simp				    ~arm_dcache_align_mask), _tmp_cl,
1192246713Skib				    buf & arm_dcache_align_mask);
1193246713Skib			if ((buf + len) & arm_dcache_align_mask)
1194246713Skib				memcpy((void *)(buf + len),
1195236991Simp				    _tmp_clend, arm_dcache_align -
1196246713Skib				    ((buf + len) & arm_dcache_align_mask));
1197234561Smarius			intr_restore(s);
1198234561Smarius		}
1199135644Scognet	}
1200135644Scognet}
1201135644Scognet
1202166063Scognetstatic void
1203166063Scognet_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1204166063Scognet{
1205166063Scognet	struct bounce_page *bpage;
1206166063Scognet
1207166063Scognet	STAILQ_FOREACH(bpage, &map->bpages, links) {
1208166063Scognet		if (op & BUS_DMASYNC_PREWRITE) {
1209246713Skib			if (bpage->datavaddr != 0)
1210259335Sian				bcopy((void *)bpage->datavaddr,
1211259335Sian				    (void *)bpage->vaddr, bpage->datacount);
1212246713Skib			else
1213246713Skib				physcopyout(bpage->dataaddr,
1214259335Sian				    (void *)bpage->vaddr,bpage->datacount);
1215259335Sian			cpu_dcache_wb_range(bpage->vaddr, bpage->datacount);
1216259335Sian			cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount);
1217187911Sthompsa			dmat->bounce_zone->total_bounced++;
1218166063Scognet		}
1219166063Scognet		if (op & BUS_DMASYNC_POSTREAD) {
1220259335Sian			cpu_dcache_inv_range(bpage->vaddr, bpage->datacount);
1221259335Sian			cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount);
1222246713Skib			if (bpage->datavaddr != 0)
1223259335Sian				bcopy((void *)bpage->vaddr,
1224246713Skib				    (void *)bpage->datavaddr, bpage->datacount);
1225246713Skib			else
1226259335Sian				physcopyin((void *)bpage->vaddr,
1227246713Skib				    bpage->dataaddr, bpage->datacount);
1228187911Sthompsa			dmat->bounce_zone->total_bounced++;
1229166063Scognet		}
1230166063Scognet	}
1231166063Scognet}
1232166063Scognet
1233129198Scognetvoid
1234143655Sjmg_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1235129198Scognet{
1236246713Skib	struct sync_list *sl, *end;
1237244471Scognet	int bufaligned;
1238244471Scognet
1239159107Scognet	if (op == BUS_DMASYNC_POSTWRITE)
1240129198Scognet		return;
1241244471Scognet	if (map->flags & DMAMAP_COHERENT)
1242244471Scognet		goto drain;
1243166063Scognet	if (STAILQ_FIRST(&map->bpages))
1244166063Scognet		_bus_dmamap_sync_bp(dmat, map, op);
1245143294Smux	CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
1246244471Scognet	bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED);
1247246713Skib	if (map->sync_count) {
1248246713Skib		end = &map->slist[map->sync_count];
1249246713Skib		for (sl = &map->slist[0]; sl != end; sl++)
1250246713Skib			bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,
1251244471Scognet			    bufaligned);
1252129198Scognet	}
1253244471Scognet
1254244471Scognetdrain:
1255244471Scognet
1256135644Scognet	cpu_drain_writebuf();
1257129198Scognet}
1258166063Scognet
1259166063Scognetstatic void
1260166063Scognetinit_bounce_pages(void *dummy __unused)
1261166063Scognet{
1262166063Scognet
1263166063Scognet	total_bpages = 0;
1264166063Scognet	STAILQ_INIT(&bounce_zone_list);
1265166063Scognet	STAILQ_INIT(&bounce_map_waitinglist);
1266166063Scognet	STAILQ_INIT(&bounce_map_callbacklist);
1267166063Scognet	mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
1268166063Scognet}
1269166063ScognetSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
1270166063Scognet
1271166063Scognetstatic struct sysctl_ctx_list *
1272166063Scognetbusdma_sysctl_tree(struct bounce_zone *bz)
1273166063Scognet{
1274166063Scognet	return (&bz->sysctl_tree);
1275166063Scognet}
1276166063Scognet
1277166063Scognetstatic struct sysctl_oid *
1278166063Scognetbusdma_sysctl_tree_top(struct bounce_zone *bz)
1279166063Scognet{
1280166063Scognet	return (bz->sysctl_tree_top);
1281166063Scognet}
1282166063Scognet
1283166063Scognetstatic int
1284166063Scognetalloc_bounce_zone(bus_dma_tag_t dmat)
1285166063Scognet{
1286166063Scognet	struct bounce_zone *bz;
1287166063Scognet
1288166063Scognet	/* Check to see if we already have a suitable zone */
1289166063Scognet	STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1290166063Scognet		if ((dmat->alignment <= bz->alignment)
1291166063Scognet		 && (dmat->lowaddr >= bz->lowaddr)) {
1292166063Scognet			dmat->bounce_zone = bz;
1293166063Scognet			return (0);
1294166063Scognet		}
1295166063Scognet	}
1296166063Scognet
1297166063Scognet	if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
1298166063Scognet	    M_NOWAIT | M_ZERO)) == NULL)
1299166063Scognet		return (ENOMEM);
1300166063Scognet
1301166063Scognet	STAILQ_INIT(&bz->bounce_page_list);
1302166063Scognet	bz->free_bpages = 0;
1303166063Scognet	bz->reserved_bpages = 0;
1304166063Scognet	bz->active_bpages = 0;
1305166063Scognet	bz->lowaddr = dmat->lowaddr;
1306191438Sjhb	bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1307188403Scognet	bz->map_count = 0;
1308166063Scognet	snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1309166063Scognet	busdma_zonecount++;
1310166063Scognet	snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1311166063Scognet	STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1312166063Scognet	dmat->bounce_zone = bz;
1313166063Scognet
1314166063Scognet	sysctl_ctx_init(&bz->sysctl_tree);
1315166063Scognet	bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1316166063Scognet	    SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1317166063Scognet	    CTLFLAG_RD, 0, "");
1318166063Scognet	if (bz->sysctl_tree_top == NULL) {
1319166063Scognet		sysctl_ctx_free(&bz->sysctl_tree);
1320166063Scognet		return (0);	/* XXX error code? */
1321166063Scognet	}
1322166063Scognet
1323166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1324166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1325166063Scognet	    "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1326166063Scognet	    "Total bounce pages");
1327166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1328166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1329166063Scognet	    "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1330166063Scognet	    "Free bounce pages");
1331166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1332166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1333166063Scognet	    "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1334166063Scognet	    "Reserved bounce pages");
1335166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1336166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1337166063Scognet	    "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1338166063Scognet	    "Active bounce pages");
1339166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1340166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1341166063Scognet	    "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1342166063Scognet	    "Total bounce requests");
1343166063Scognet	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1344166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1345166063Scognet	    "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1346166063Scognet	    "Total bounce requests that were deferred");
1347166063Scognet	SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1348166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1349166063Scognet	    "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1350273736Shselasky	SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz),
1351166063Scognet	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1352273736Shselasky	    "alignment", CTLFLAG_RD, &bz->alignment, "");
1353166063Scognet
1354166063Scognet	return (0);
1355166063Scognet}
1356166063Scognet
1357166063Scognetstatic int
1358166063Scognetalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1359166063Scognet{
1360166063Scognet	struct bounce_zone *bz;
1361166063Scognet	int count;
1362166063Scognet
1363166063Scognet	bz = dmat->bounce_zone;
1364166063Scognet	count = 0;
1365166063Scognet	while (numpages > 0) {
1366166063Scognet		struct bounce_page *bpage;
1367166063Scognet
1368166063Scognet		bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1369166063Scognet						     M_NOWAIT | M_ZERO);
1370166063Scognet
1371166063Scognet		if (bpage == NULL)
1372166063Scognet			break;
1373166063Scognet		bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1374166063Scognet							 M_NOWAIT, 0ul,
1375166063Scognet							 bz->lowaddr,
1376166063Scognet							 PAGE_SIZE,
1377191438Sjhb							 0);
1378166063Scognet		if (bpage->vaddr == 0) {
1379166063Scognet			free(bpage, M_DEVBUF);
1380166063Scognet			break;
1381166063Scognet		}
1382166063Scognet		bpage->busaddr = pmap_kextract(bpage->vaddr);
1383166063Scognet		mtx_lock(&bounce_lock);
1384166063Scognet		STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1385166063Scognet		total_bpages++;
1386166063Scognet		bz->total_bpages++;
1387166063Scognet		bz->free_bpages++;
1388166063Scognet		mtx_unlock(&bounce_lock);
1389166063Scognet		count++;
1390166063Scognet		numpages--;
1391166063Scognet	}
1392166063Scognet	return (count);
1393166063Scognet}
1394166063Scognet
1395166063Scognetstatic int
1396166063Scognetreserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1397166063Scognet{
1398166063Scognet	struct bounce_zone *bz;
1399166063Scognet	int pages;
1400166063Scognet
1401166063Scognet	mtx_assert(&bounce_lock, MA_OWNED);
1402166063Scognet	bz = dmat->bounce_zone;
1403166063Scognet	pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1404166063Scognet	if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1405166063Scognet		return (map->pagesneeded - (map->pagesreserved + pages));
1406166063Scognet	bz->free_bpages -= pages;
1407166063Scognet	bz->reserved_bpages += pages;
1408166063Scognet	map->pagesreserved += pages;
1409166063Scognet	pages = map->pagesneeded - map->pagesreserved;
1410166063Scognet
1411166063Scognet	return (pages);
1412166063Scognet}
1413166063Scognet
1414166063Scognetstatic bus_addr_t
1415166063Scognetadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1416246713Skib		bus_addr_t addr, bus_size_t size)
1417166063Scognet{
1418166063Scognet	struct bounce_zone *bz;
1419166063Scognet	struct bounce_page *bpage;
1420166063Scognet
1421166063Scognet	KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1422170406Scognet	KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
1423166063Scognet
1424166063Scognet	bz = dmat->bounce_zone;
1425166063Scognet	if (map->pagesneeded == 0)
1426166063Scognet		panic("add_bounce_page: map doesn't need any pages");
1427166063Scognet	map->pagesneeded--;
1428166063Scognet
1429166063Scognet	if (map->pagesreserved == 0)
1430166063Scognet		panic("add_bounce_page: map doesn't need any pages");
1431166063Scognet	map->pagesreserved--;
1432166063Scognet
1433166063Scognet	mtx_lock(&bounce_lock);
1434166063Scognet	bpage = STAILQ_FIRST(&bz->bounce_page_list);
1435166063Scognet	if (bpage == NULL)
1436166063Scognet		panic("add_bounce_page: free page list is empty");
1437166063Scognet
1438166063Scognet	STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1439166063Scognet	bz->reserved_bpages--;
1440166063Scognet	bz->active_bpages++;
1441166063Scognet	mtx_unlock(&bounce_lock);
1442166063Scognet
1443188350Simp	if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1444191201Sjhb		/* Page offset needs to be preserved. */
1445282506Shselasky		bpage->vaddr |= addr & PAGE_MASK;
1446282506Shselasky		bpage->busaddr |= addr & PAGE_MASK;
1447188350Simp	}
1448166063Scognet	bpage->datavaddr = vaddr;
1449246713Skib	bpage->dataaddr = addr;
1450166063Scognet	bpage->datacount = size;
1451166063Scognet	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1452166063Scognet	return (bpage->busaddr);
1453166063Scognet}
1454166063Scognet
1455166063Scognetstatic void
1456166063Scognetfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1457166063Scognet{
1458166063Scognet	struct bus_dmamap *map;
1459166063Scognet	struct bounce_zone *bz;
1460166063Scognet
1461166063Scognet	bz = dmat->bounce_zone;
1462166063Scognet	bpage->datavaddr = 0;
1463166063Scognet	bpage->datacount = 0;
1464191201Sjhb	if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1465191201Sjhb		/*
1466191201Sjhb		 * Reset the bounce page to start at offset 0.  Other uses
1467191201Sjhb		 * of this bounce page may need to store a full page of
1468191201Sjhb		 * data and/or assume it starts on a page boundary.
1469191201Sjhb		 */
1470191201Sjhb		bpage->vaddr &= ~PAGE_MASK;
1471191201Sjhb		bpage->busaddr &= ~PAGE_MASK;
1472191201Sjhb	}
1473166063Scognet
1474166063Scognet	mtx_lock(&bounce_lock);
1475166063Scognet	STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1476166063Scognet	bz->free_bpages++;
1477166063Scognet	bz->active_bpages--;
1478166063Scognet	if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1479166063Scognet		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1480166063Scognet			STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1481166063Scognet			STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1482166063Scognet					   map, links);
1483166063Scognet			busdma_swi_pending = 1;
1484166063Scognet			bz->total_deferred++;
1485166063Scognet			swi_sched(vm_ih, 0);
1486166063Scognet		}
1487166063Scognet	}
1488166063Scognet	mtx_unlock(&bounce_lock);
1489166063Scognet}
1490166063Scognet
1491166063Scognetvoid
1492166063Scognetbusdma_swi(void)
1493166063Scognet{
1494166063Scognet	bus_dma_tag_t dmat;
1495166063Scognet	struct bus_dmamap *map;
1496166063Scognet
1497166063Scognet	mtx_lock(&bounce_lock);
1498166063Scognet	while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1499166063Scognet		STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1500166063Scognet		mtx_unlock(&bounce_lock);
1501166063Scognet		dmat = map->dmat;
1502166063Scognet		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1503246713Skib		bus_dmamap_load_mem(map->dmat, map, &map->mem,
1504246713Skib		    map->callback, map->callback_arg, BUS_DMA_WAITOK);
1505166063Scognet		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1506166063Scognet		mtx_lock(&bounce_lock);
1507166063Scognet	}
1508166063Scognet	mtx_unlock(&bounce_lock);
1509166063Scognet}
1510