busdma_machdep.c revision 117126 
1/*
2 * Copyright (c) 1997, 1998 Justin T. Gibbs.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions, and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. The name of the author may not be used to endorse or promote products
12 *    derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/i386/i386/busdma_machdep.c 117126 2003-07-01 15:52:06Z scottl $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/malloc.h>
34#include <sys/bus.h>
35#include <sys/interrupt.h>
36#include <sys/kernel.h>
37#include <sys/lock.h>
38#include <sys/proc.h>
39#include <sys/mutex.h>
40#include <sys/mbuf.h>
41#include <sys/uio.h>
42
43#include <vm/vm.h>
44#include <vm/vm_page.h>
45#include <vm/vm_map.h>
46
47#include <machine/atomic.h>
48#include <machine/bus.h>
49#include <machine/md_var.h>
50
51#define MAX_BPAGES 512
52
53struct bus_dma_tag {
54	bus_dma_tag_t	  parent;
55	bus_size_t	  alignment;
56	bus_size_t	  boundary;
57	bus_addr_t	  lowaddr;
58	bus_addr_t	  highaddr;
59	bus_dma_filter_t *filter;
60	void		 *filterarg;
61	bus_size_t	  maxsize;
62	u_int		  nsegments;
63	bus_size_t	  maxsegsz;
64	int		  flags;
65	int		  ref_count;
66	int		  map_count;
67	bus_dma_lock_t	 *lockfunc;
68	void		 *lockfuncarg;
69};
70
71struct bounce_page {
72	vm_offset_t	vaddr;		/* kva of bounce buffer */
73	bus_addr_t	busaddr;	/* Physical address */
74	vm_offset_t	datavaddr;	/* kva of client data */
75	bus_size_t	datacount;	/* client data count */
76	STAILQ_ENTRY(bounce_page) links;
77};
78
79int busdma_swi_pending;
80
81static STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
82static int free_bpages;
83static int reserved_bpages;
84static int active_bpages;
85static int total_bpages;
86static bus_addr_t bounce_lowaddr = BUS_SPACE_MAXADDR;
87
88struct bus_dmamap {
89	struct bp_list	       bpages;
90	int		       pagesneeded;
91	int		       pagesreserved;
92	bus_dma_tag_t	       dmat;
93	void		      *buf;		/* unmapped buffer pointer */
94	bus_size_t	       buflen;		/* unmapped buffer length */
95	bus_dmamap_callback_t *callback;
96	void		      *callback_arg;
97	STAILQ_ENTRY(bus_dmamap) links;
98};
99
100static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
101static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
102static struct bus_dmamap nobounce_dmamap;
103
104static void init_bounce_pages(void *dummy);
105static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
106static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
107    				int commit);
108static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
109				   vm_offset_t vaddr, bus_size_t size);
110static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
111static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
112
113/* To protect all the the bounce pages related lists and data. */
114static struct mtx bounce_lock;
115
116/*
117 * Return true if a match is made.
118 *
119 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
120 *
121 * If paddr is within the bounds of the dma tag then call the filter callback
122 * to check for a match, if there is no filter callback then assume a match.
123 */
124static __inline int
125run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
126{
127	int retval;
128
129	retval = 0;
130	do {
131		if (paddr > dmat->lowaddr
132		 && paddr <= dmat->highaddr
133		 && (dmat->filter == NULL
134		  || (*dmat->filter)(dmat->filterarg, paddr) != 0))
135			retval = 1;
136
137		dmat = dmat->parent;
138	} while (retval == 0 && dmat != NULL);
139	return (retval);
140}
141
142/*
143 * Convenience function for manipulating driver locks from busdma (during
144 * busdma_swi, for example).  Drivers that don't provide their own locks
145 * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
146 * non-mutex locking scheme don't have to use this at all.
147 */
148void
149busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
150{
151	struct mtx *dmtx;
152
153	dmtx = (struct mtx *)arg;
154	switch (op) {
155	case BUS_DMA_LOCK:
156		mtx_lock(dmtx);
157		break;
158	case BUS_DMA_UNLOCK:
159		mtx_unlock(dmtx);
160		break;
161	default:
162		panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
163	}
164}
165
166/*
167 * dflt_lock should never get called.  It gets put into the dma tag when
168 * lockfunc == NULL, which is only valid if the maps that are associated
169 * with the tag are meant to never be defered.
170 * XXX Should have a way to identify which driver is responsible here.
171 */
172static void
173dflt_lock(void *arg, bus_dma_lock_op_t op)
174{
175#ifdef INVARIANTS
176	panic("driver error: busdma dflt_lock called");
177#else
178	printf("DRIVER_ERROR: busdma dflt_lock called\n");
179#endif
180}
181
182#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
183/*
184 * Allocate a device specific dma_tag.
185 */
186int
187bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
188		   bus_size_t boundary, bus_addr_t lowaddr,
189		   bus_addr_t highaddr, bus_dma_filter_t *filter,
190		   void *filterarg, bus_size_t maxsize, int nsegments,
191		   bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
192		   void *lockfuncarg, bus_dma_tag_t *dmat)
193{
194	bus_dma_tag_t newtag;
195	int error = 0;
196
197	/* Return a NULL tag on failure */
198	*dmat = NULL;
199
200	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
201	if (newtag == NULL)
202		return (ENOMEM);
203
204	newtag->parent = parent;
205	newtag->alignment = alignment;
206	newtag->boundary = boundary;
207	newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
208	newtag->highaddr = trunc_page((vm_paddr_t)highaddr) +
209	    (PAGE_SIZE - 1);
210	newtag->filter = filter;
211	newtag->filterarg = filterarg;
212	newtag->maxsize = maxsize;
213	newtag->nsegments = nsegments;
214	newtag->maxsegsz = maxsegsz;
215	newtag->flags = flags;
216	newtag->ref_count = 1; /* Count ourself */
217	newtag->map_count = 0;
218	if (lockfunc != NULL) {
219		newtag->lockfunc = lockfunc;
220		newtag->lockfuncarg = lockfuncarg;
221	} else {
222		newtag->lockfunc = dflt_lock;
223		newtag->lockfuncarg = NULL;
224	}
225
226	/* Take into account any restrictions imposed by our parent tag */
227	if (parent != NULL) {
228		newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
229		newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
230		/*
231		 * XXX Not really correct??? Probably need to honor boundary
232		 *     all the way up the inheritence chain.
233		 */
234		newtag->boundary = MAX(parent->boundary, newtag->boundary);
235		if (newtag->filter == NULL) {
236			/*
237			 * Short circuit looking at our parent directly
238			 * since we have encapsulated all of its information
239			 */
240			newtag->filter = parent->filter;
241			newtag->filterarg = parent->filterarg;
242			newtag->parent = parent->parent;
243		}
244		if (newtag->parent != NULL)
245			atomic_add_int(&parent->ref_count, 1);
246	}
247
248	if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) &&
249	    (flags & BUS_DMA_ALLOCNOW) != 0) {
250		/* Must bounce */
251
252		if (lowaddr > bounce_lowaddr) {
253			/*
254			 * Go through the pool and kill any pages
255			 * that don't reside below lowaddr.
256			 */
257			panic("bus_dma_tag_create: page reallocation "
258			      "not implemented");
259		}
260		if (ptoa(total_bpages) < maxsize) {
261			int pages;
262
263			pages = atop(maxsize) - total_bpages;
264
265			/* Add pages to our bounce pool */
266			if (alloc_bounce_pages(newtag, pages) < pages)
267				error = ENOMEM;
268		}
269		/* Performed initial allocation */
270		newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
271	}
272
273	if (error != 0) {
274		free(newtag, M_DEVBUF);
275	} else {
276		*dmat = newtag;
277	}
278	return (error);
279}
280
281int
282bus_dma_tag_destroy(bus_dma_tag_t dmat)
283{
284	if (dmat != NULL) {
285
286		if (dmat->map_count != 0)
287			return (EBUSY);
288
289		while (dmat != NULL) {
290			bus_dma_tag_t parent;
291
292			parent = dmat->parent;
293			atomic_subtract_int(&dmat->ref_count, 1);
294			if (dmat->ref_count == 0) {
295				free(dmat, M_DEVBUF);
296				/*
297				 * Last reference count, so
298				 * release our reference
299				 * count on our parent.
300				 */
301				dmat = parent;
302			} else
303				dmat = NULL;
304		}
305	}
306	return (0);
307}
308
309/*
310 * Allocate a handle for mapping from kva/uva/physical
311 * address space into bus device space.
312 */
313int
314bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
315{
316	int error;
317
318	error = 0;
319
320	if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem)) {
321		/* Must bounce */
322		int maxpages;
323
324		*mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
325					     M_NOWAIT | M_ZERO);
326		if (*mapp == NULL)
327			return (ENOMEM);
328
329		/* Initialize the new map */
330		STAILQ_INIT(&((*mapp)->bpages));
331
332		/*
333		 * Attempt to add pages to our pool on a per-instance
334		 * basis up to a sane limit.
335		 */
336		maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->lowaddr));
337		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
338		 || (dmat->map_count > 0
339		  && total_bpages < maxpages)) {
340			int pages;
341
342			if (dmat->lowaddr > bounce_lowaddr) {
343				/*
344				 * Go through the pool and kill any pages
345				 * that don't reside below lowaddr.
346				 */
347				panic("bus_dmamap_create: page reallocation "
348				      "not implemented");
349			}
350			pages = MAX(atop(dmat->maxsize), 1);
351			pages = MIN(maxpages - total_bpages, pages);
352			if (alloc_bounce_pages(dmat, pages) < pages)
353				error = ENOMEM;
354
355			if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
356				if (error == 0)
357					dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
358			} else {
359				error = 0;
360			}
361		}
362	} else {
363		*mapp = NULL;
364	}
365	if (error == 0)
366		dmat->map_count++;
367	return (error);
368}
369
370/*
371 * Destroy a handle for mapping from kva/uva/physical
372 * address space into bus device space.
373 */
374int
375bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
376{
377	if (map != NULL) {
378		if (STAILQ_FIRST(&map->bpages) != NULL)
379			return (EBUSY);
380		free(map, M_DEVBUF);
381	}
382	dmat->map_count--;
383	return (0);
384}
385
386
387/*
388 * Allocate a piece of memory that can be efficiently mapped into
389 * bus device space based on the constraints lited in the dma tag.
390 * A dmamap to for use with dmamap_load is also allocated.
391 */
392int
393bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
394		 bus_dmamap_t *mapp)
395{
396	/* If we succeed, no mapping/bouncing will be required */
397	*mapp = NULL;
398
399	if ((dmat->maxsize <= PAGE_SIZE) &&
400	    dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) {
401		*vaddr = malloc(dmat->maxsize, M_DEVBUF,
402				(flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK);
403	} else {
404		/*
405		 * XXX Use Contigmalloc until it is merged into this facility
406		 *     and handles multi-seg allocations.  Nobody is doing
407		 *     multi-seg allocations yet though.
408		 */
409		mtx_lock(&Giant);
410		*vaddr = contigmalloc(dmat->maxsize, M_DEVBUF,
411		    (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK,
412		    0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
413		    dmat->boundary);
414		mtx_unlock(&Giant);
415	}
416	if (*vaddr == NULL)
417		return (ENOMEM);
418	return (0);
419}
420
421/*
422 * Free a piece of memory and it's allociated dmamap, that was allocated
423 * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
424 */
425void
426bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
427{
428	/*
429	 * dmamem does not need to be bounced, so the map should be
430	 * NULL
431	 */
432	if (map != NULL)
433		panic("bus_dmamem_free: Invalid map freed\n");
434	if ((dmat->maxsize <= PAGE_SIZE)
435	 && dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem))
436		free(vaddr, M_DEVBUF);
437	else {
438		mtx_lock(&Giant);
439		contigfree(vaddr, dmat->maxsize, M_DEVBUF);
440		mtx_unlock(&Giant);
441	}
442}
443
444/*
445 * Utility function to load a linear buffer.  lastaddrp holds state
446 * between invocations (for multiple-buffer loads).  segp contains
447 * the starting segment on entrace, and the ending segment on exit.
448 * first indicates if this is the first invocation of this function.
449 */
450static int
451_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
452    			bus_dmamap_t map,
453			bus_dma_segment_t segs[],
454			void *buf, bus_size_t buflen,
455			struct thread *td,
456			int flags,
457			bus_addr_t *lastaddrp,
458			int *segp,
459			int first)
460{
461	bus_size_t sgsize;
462	bus_addr_t curaddr, lastaddr, baddr, bmask;
463	vm_offset_t vaddr;
464	bus_addr_t paddr;
465	int needbounce = 0;
466	int seg;
467	pmap_t pmap;
468
469	if (map == NULL)
470		map = &nobounce_dmamap;
471
472	if (td != NULL)
473		pmap = vmspace_pmap(td->td_proc->p_vmspace);
474	else
475		pmap = NULL;
476
477	if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem)) {
478		vm_offset_t	vendaddr;
479
480		/*
481		 * Count the number of bounce pages
482		 * needed in order to complete this transfer
483		 */
484		vaddr = trunc_page((vm_offset_t)buf);
485		vendaddr = (vm_offset_t)buf + buflen;
486
487		while (vaddr < vendaddr) {
488			paddr = pmap_kextract(vaddr);
489			if (run_filter(dmat, paddr) != 0) {
490				needbounce = 1;
491				map->pagesneeded++;
492			}
493			vaddr += PAGE_SIZE;
494		}
495	}
496
497	vaddr = (vm_offset_t)buf;
498
499	/* Reserve Necessary Bounce Pages */
500	if (map->pagesneeded != 0) {
501		mtx_lock(&bounce_lock);
502		if (flags & BUS_DMA_NOWAIT) {
503			if (reserve_bounce_pages(dmat, map, 0) != 0) {
504				mtx_unlock(&bounce_lock);
505				return (ENOMEM);
506			}
507		} else {
508			if (reserve_bounce_pages(dmat, map, 1) != 0) {
509				/* Queue us for resources */
510				map->dmat = dmat;
511				map->buf = buf;
512				map->buflen = buflen;
513				STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
514								map, links);
515				mtx_unlock(&bounce_lock);
516				return (EINPROGRESS);
517			}
518		}
519		mtx_unlock(&bounce_lock);
520	}
521
522	lastaddr = *lastaddrp;
523	bmask = ~(dmat->boundary - 1);
524
525	for (seg = *segp; buflen > 0 ; ) {
526		/*
527		 * Get the physical address for this segment.
528		 */
529		if (pmap)
530			curaddr = pmap_extract(pmap, vaddr);
531		else
532			curaddr = pmap_kextract(vaddr);
533
534		/*
535		 * Compute the segment size, and adjust counts.
536		 */
537		sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
538		if (buflen < sgsize)
539			sgsize = buflen;
540
541		/*
542		 * Make sure we don't cross any boundaries.
543		 */
544		if (dmat->boundary > 0) {
545			baddr = (curaddr + dmat->boundary) & bmask;
546			if (sgsize > (baddr - curaddr))
547				sgsize = (baddr - curaddr);
548		}
549
550		if (map->pagesneeded != 0 && run_filter(dmat, curaddr))
551			curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
552
553		/*
554		 * Insert chunk into a segment, coalescing with
555		 * previous segment if possible.
556		 */
557		if (first) {
558			segs[seg].ds_addr = curaddr;
559			segs[seg].ds_len = sgsize;
560			first = 0;
561		} else {
562			if (needbounce == 0 && curaddr == lastaddr &&
563			    (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
564			    (dmat->boundary == 0 ||
565			     (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
566				segs[seg].ds_len += sgsize;
567			else {
568				if (++seg >= dmat->nsegments)
569					break;
570				segs[seg].ds_addr = curaddr;
571				segs[seg].ds_len = sgsize;
572			}
573		}
574
575		lastaddr = curaddr + sgsize;
576		vaddr += sgsize;
577		buflen -= sgsize;
578	}
579
580	*segp = seg;
581	*lastaddrp = lastaddr;
582
583	/*
584	 * Did we fit?
585	 */
586	return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
587}
588
589#define BUS_DMAMAP_NSEGS ((64 * 1024) / PAGE_SIZE + 1)
590
591/*
592 * Map the buffer buf into bus space using the dmamap map.
593 */
594int
595bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
596		bus_size_t buflen, bus_dmamap_callback_t *callback,
597		void *callback_arg, int flags)
598{
599#ifdef __GNUC__
600	bus_dma_segment_t	dm_segments[dmat->nsegments];
601#else
602	bus_dma_segment_t	dm_segments[BUS_DMAMAP_NSEGS];
603#endif
604	bus_addr_t		lastaddr = 0;
605	int			error, nsegs = 0;
606
607	if (map != NULL) {
608		flags |= BUS_DMA_WAITOK;
609		map->callback = callback;
610		map->callback_arg = callback_arg;
611	}
612
613	error = _bus_dmamap_load_buffer(dmat, map, dm_segments, buf, buflen,
614	    NULL, flags, &lastaddr, &nsegs, 1);
615
616	if (error == EINPROGRESS)
617		return (error);
618
619	if (error)
620		(*callback)(callback_arg, dm_segments, 0, error);
621	else
622		(*callback)(callback_arg, dm_segments, nsegs + 1, 0);
623
624	return (0);
625}
626
627
628/*
629 * Like _bus_dmamap_load(), but for mbufs.
630 */
631int
632bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
633		     struct mbuf *m0,
634		     bus_dmamap_callback2_t *callback, void *callback_arg,
635		     int flags)
636{
637#ifdef __GNUC__
638	bus_dma_segment_t dm_segments[dmat->nsegments];
639#else
640	bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS];
641#endif
642	int nsegs, error;
643
644	KASSERT(m0->m_flags & M_PKTHDR,
645		("bus_dmamap_load_mbuf: no packet header"));
646
647	flags |= BUS_DMA_NOWAIT;
648	nsegs = 0;
649	error = 0;
650	if (m0->m_pkthdr.len <= dmat->maxsize) {
651		int first = 1;
652		bus_addr_t lastaddr = 0;
653		struct mbuf *m;
654
655		for (m = m0; m != NULL && error == 0; m = m->m_next) {
656			if (m->m_len > 0) {
657				error = _bus_dmamap_load_buffer(dmat, map,
658						dm_segments,
659						m->m_data, m->m_len,
660						NULL, flags, &lastaddr,
661						&nsegs, first);
662				first = 0;
663			}
664		}
665	} else {
666		error = EINVAL;
667	}
668
669	if (error) {
670		/* force "no valid mappings" in callback */
671		(*callback)(callback_arg, dm_segments, 0, 0, error);
672	} else {
673		(*callback)(callback_arg, dm_segments,
674			    nsegs+1, m0->m_pkthdr.len, error);
675	}
676	return (error);
677}
678
679/*
680 * Like _bus_dmamap_load(), but for uios.
681 */
682int
683bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
684		    struct uio *uio,
685		    bus_dmamap_callback2_t *callback, void *callback_arg,
686		    int flags)
687{
688	bus_addr_t lastaddr;
689#ifdef __GNUC__
690	bus_dma_segment_t dm_segments[dmat->nsegments];
691#else
692	bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS];
693#endif
694	int nsegs, error, first, i;
695	bus_size_t resid;
696	struct iovec *iov;
697	struct thread *td = NULL;
698
699	flags |= BUS_DMA_NOWAIT;
700	resid = uio->uio_resid;
701	iov = uio->uio_iov;
702
703	if (uio->uio_segflg == UIO_USERSPACE) {
704		td = uio->uio_td;
705		KASSERT(td != NULL,
706			("bus_dmamap_load_uio: USERSPACE but no proc"));
707	}
708
709	nsegs = 0;
710	error = 0;
711	first = 1;
712	for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
713		/*
714		 * Now at the first iovec to load.  Load each iovec
715		 * until we have exhausted the residual count.
716		 */
717		bus_size_t minlen =
718			resid < iov[i].iov_len ? resid : iov[i].iov_len;
719		caddr_t addr = (caddr_t) iov[i].iov_base;
720
721		if (minlen > 0) {
722			error = _bus_dmamap_load_buffer(dmat, map,
723					dm_segments,
724					addr, minlen,
725					td, flags, &lastaddr, &nsegs, first);
726			first = 0;
727
728			resid -= minlen;
729		}
730	}
731
732	if (error) {
733		/* force "no valid mappings" in callback */
734		(*callback)(callback_arg, dm_segments, 0, 0, error);
735	} else {
736		(*callback)(callback_arg, dm_segments,
737			    nsegs+1, uio->uio_resid, error);
738	}
739	return (error);
740}
741
742/*
743 * Release the mapping held by map.
744 */
745void
746_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
747{
748	struct bounce_page *bpage;
749
750	while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
751		STAILQ_REMOVE_HEAD(&map->bpages, links);
752		free_bounce_page(dmat, bpage);
753	}
754}
755
756void
757_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
758{
759	struct bounce_page *bpage;
760
761	if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
762		/*
763		 * Handle data bouncing.  We might also
764		 * want to add support for invalidating
765		 * the caches on broken hardware
766		 */
767		if (op & BUS_DMASYNC_PREWRITE) {
768			while (bpage != NULL) {
769				bcopy((void *)bpage->datavaddr,
770				      (void *)bpage->vaddr,
771				      bpage->datacount);
772				bpage = STAILQ_NEXT(bpage, links);
773			}
774		}
775
776		if (op & BUS_DMASYNC_POSTREAD) {
777			while (bpage != NULL) {
778				bcopy((void *)bpage->vaddr,
779				      (void *)bpage->datavaddr,
780				      bpage->datacount);
781				bpage = STAILQ_NEXT(bpage, links);
782			}
783		}
784	}
785}
786
787static void
788init_bounce_pages(void *dummy __unused)
789{
790
791	free_bpages = 0;
792	reserved_bpages = 0;
793	active_bpages = 0;
794	total_bpages = 0;
795	STAILQ_INIT(&bounce_page_list);
796	STAILQ_INIT(&bounce_map_waitinglist);
797	STAILQ_INIT(&bounce_map_callbacklist);
798	mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
799}
800SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
801
802static int
803alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
804{
805	int count;
806
807	count = 0;
808	while (numpages > 0) {
809		struct bounce_page *bpage;
810
811		bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
812						     M_NOWAIT | M_ZERO);
813
814		if (bpage == NULL)
815			break;
816		mtx_lock(&Giant);
817		bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
818							 M_NOWAIT, 0ul,
819							 dmat->lowaddr,
820							 PAGE_SIZE,
821							 0);
822		mtx_unlock(&Giant);
823		if (bpage->vaddr == 0) {
824			free(bpage, M_DEVBUF);
825			break;
826		}
827		bpage->busaddr = pmap_kextract(bpage->vaddr);
828		mtx_lock(&bounce_lock);
829		STAILQ_INSERT_TAIL(&bounce_page_list, bpage, links);
830		total_bpages++;
831		free_bpages++;
832		mtx_unlock(&bounce_lock);
833		count++;
834		numpages--;
835	}
836	return (count);
837}
838
839static int
840reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
841{
842	int pages;
843
844	mtx_assert(&bounce_lock, MA_OWNED);
845	pages = MIN(free_bpages, map->pagesneeded - map->pagesreserved);
846	if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
847		return (map->pagesneeded - (map->pagesreserved + pages));
848	free_bpages -= pages;
849	reserved_bpages += pages;
850	map->pagesreserved += pages;
851	pages = map->pagesneeded - map->pagesreserved;
852
853	return (pages);
854}
855
856static bus_addr_t
857add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
858		bus_size_t size)
859{
860	struct bounce_page *bpage;
861
862	KASSERT(map != NULL && map != &nobounce_dmamap,
863	    ("add_bounce_page: bad map %p", map));
864
865	if (map->pagesneeded == 0)
866		panic("add_bounce_page: map doesn't need any pages");
867	map->pagesneeded--;
868
869	if (map->pagesreserved == 0)
870		panic("add_bounce_page: map doesn't need any pages");
871	map->pagesreserved--;
872
873	mtx_lock(&bounce_lock);
874	bpage = STAILQ_FIRST(&bounce_page_list);
875	if (bpage == NULL)
876		panic("add_bounce_page: free page list is empty");
877
878	STAILQ_REMOVE_HEAD(&bounce_page_list, links);
879	reserved_bpages--;
880	active_bpages++;
881	mtx_unlock(&bounce_lock);
882
883	bpage->datavaddr = vaddr;
884	bpage->datacount = size;
885	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
886	return (bpage->busaddr);
887}
888
889static void
890free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
891{
892	struct bus_dmamap *map;
893
894	bpage->datavaddr = 0;
895	bpage->datacount = 0;
896
897	mtx_lock(&bounce_lock);
898	STAILQ_INSERT_HEAD(&bounce_page_list, bpage, links);
899	free_bpages++;
900	active_bpages--;
901	if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
902		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
903			STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
904			STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
905					   map, links);
906			busdma_swi_pending = 1;
907			swi_sched(vm_ih, 0);
908		}
909	}
910	mtx_unlock(&bounce_lock);
911}
912
913void
914busdma_swi(void)
915{
916	bus_dma_tag_t dmat;
917	struct bus_dmamap *map;
918
919	mtx_lock(&bounce_lock);
920	while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
921		STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
922		mtx_unlock(&bounce_lock);
923		dmat = (map)->dmat;
924		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
925		bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
926				map->callback, map->callback_arg, /*flags*/0);
927		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
928		mtx_lock(&bounce_lock);
929	}
930	mtx_unlock(&bounce_lock);
931}
932