1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 1997, 1998 Justin T. Gibbs.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions, and the following disclaimer,
12 *    without modification, immediately at the beginning of the file.
13 * 2. The name of the author may not be used to endorse or promote products
14 *    derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29/*
30 * From amd64/busdma_machdep.c, r204214
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36#include <sys/param.h>
37#include <sys/systm.h>
38#include <sys/malloc.h>
39#include <sys/bus.h>
40#include <sys/interrupt.h>
41#include <sys/kernel.h>
42#include <sys/ktr.h>
43#include <sys/lock.h>
44#include <sys/proc.h>
45#include <sys/memdesc.h>
46#include <sys/mutex.h>
47#include <sys/sysctl.h>
48#include <sys/uio.h>
49
50#include <vm/vm.h>
51#include <vm/vm_extern.h>
52#include <vm/vm_kern.h>
53#include <vm/vm_page.h>
54#include <vm/vm_map.h>
55
56#include <machine/atomic.h>
57#include <machine/bus.h>
58#include <machine/cpufunc.h>
59#include <machine/md_var.h>
60
61#include "iommu_if.h"
62
63#define MAX_BPAGES MIN(8192, physmem/40)
64
65struct bounce_zone;
66
67struct bus_dma_tag {
68	bus_dma_tag_t	  parent;
69	bus_size_t	  alignment;
70	bus_addr_t	  boundary;
71	bus_addr_t	  lowaddr;
72	bus_addr_t	  highaddr;
73	bus_dma_filter_t *filter;
74	void		 *filterarg;
75	bus_size_t	  maxsize;
76	bus_size_t	  maxsegsz;
77	u_int		  nsegments;
78	int		  flags;
79	int		  ref_count;
80	int		  map_count;
81	bus_dma_lock_t	 *lockfunc;
82	void		 *lockfuncarg;
83	struct bounce_zone *bounce_zone;
84	device_t	  iommu;
85	void		 *iommu_cookie;
86};
87
88struct bounce_page {
89	vm_offset_t	vaddr;		/* kva of bounce buffer */
90	bus_addr_t	busaddr;	/* Physical address */
91	vm_offset_t	datavaddr;	/* kva of client data */
92	vm_page_t	datapage;	/* physical page of client data */
93	vm_offset_t	dataoffs;	/* page offset of client data */
94	bus_size_t	datacount;	/* client data count */
95	STAILQ_ENTRY(bounce_page) links;
96};
97
98int busdma_swi_pending;
99
100struct bounce_zone {
101	STAILQ_ENTRY(bounce_zone) links;
102	STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
103	int		total_bpages;
104	int		free_bpages;
105	int		reserved_bpages;
106	int		active_bpages;
107	int		total_bounced;
108	int		total_deferred;
109	int		map_count;
110	bus_size_t	alignment;
111	bus_addr_t	lowaddr;
112	char		zoneid[8];
113	char		lowaddrid[20];
114	struct sysctl_ctx_list sysctl_tree;
115	struct sysctl_oid *sysctl_tree_top;
116};
117
118static struct mtx bounce_lock;
119static int total_bpages;
120static int busdma_zonecount;
121static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
122
123static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
124    "Busdma parameters");
125SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
126	   "Total bounce pages");
127
128struct bus_dmamap {
129	struct bp_list	       bpages;
130	int		       pagesneeded;
131	int		       pagesreserved;
132	bus_dma_tag_t	       dmat;
133	struct memdesc	       mem;
134	bus_dma_segment_t     *segments;
135	int		       nsegs;
136	bus_dmamap_callback_t *callback;
137	void		      *callback_arg;
138	STAILQ_ENTRY(bus_dmamap) links;
139	int		       contigalloc;
140};
141
142static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
143static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
144
145static void init_bounce_pages(void *dummy);
146static int alloc_bounce_zone(bus_dma_tag_t dmat);
147static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
148static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
149				int commit);
150static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
151				  vm_offset_t vaddr, bus_addr_t addr,
152				  bus_size_t size);
153static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
154static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
155
156/*
157 * Return true if a match is made.
158 *
159 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
160 *
161 * If paddr is within the bounds of the dma tag then call the filter callback
162 * to check for a match, if there is no filter callback then assume a match.
163 */
164static __inline int
165run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
166{
167	int retval;
168
169	retval = 0;
170
171	do {
172		if (dmat->filter == NULL && dmat->iommu == NULL &&
173		    paddr > dmat->lowaddr && paddr <= dmat->highaddr)
174			retval = 1;
175		if (dmat->filter == NULL &&
176		    (paddr & (dmat->alignment - 1)) != 0)
177			retval = 1;
178		if (dmat->filter != NULL &&
179		    (*dmat->filter)(dmat->filterarg, paddr) != 0)
180			retval = 1;
181
182		dmat = dmat->parent;
183	} while (retval == 0 && dmat != NULL);
184	return (retval);
185}
186
187/*
188 * Convenience function for manipulating driver locks from busdma (during
189 * busdma_swi, for example).  Drivers that don't provide their own locks
190 * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
191 * non-mutex locking scheme don't have to use this at all.
192 */
193void
194busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
195{
196	struct mtx *dmtx;
197
198	dmtx = (struct mtx *)arg;
199	switch (op) {
200	case BUS_DMA_LOCK:
201		mtx_lock(dmtx);
202		break;
203	case BUS_DMA_UNLOCK:
204		mtx_unlock(dmtx);
205		break;
206	default:
207		panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
208	}
209}
210
211/*
212 * dflt_lock should never get called.  It gets put into the dma tag when
213 * lockfunc == NULL, which is only valid if the maps that are associated
214 * with the tag are meant to never be defered.
215 * XXX Should have a way to identify which driver is responsible here.
216 */
217static void
218dflt_lock(void *arg, bus_dma_lock_op_t op)
219{
220	panic("driver error: busdma dflt_lock called");
221}
222
223#define BUS_DMA_COULD_BOUNCE	BUS_DMA_BUS3
224#define BUS_DMA_MIN_ALLOC_COMP	BUS_DMA_BUS4
225/*
226 * Allocate a device specific dma_tag.
227 */
228int
229bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
230		   bus_addr_t boundary, bus_addr_t lowaddr,
231		   bus_addr_t highaddr, bus_dma_filter_t *filter,
232		   void *filterarg, bus_size_t maxsize, int nsegments,
233		   bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
234		   void *lockfuncarg, bus_dma_tag_t *dmat)
235{
236	bus_dma_tag_t newtag;
237	int error = 0;
238
239	/* Basic sanity checking */
240	if (boundary != 0 && boundary < maxsegsz)
241		maxsegsz = boundary;
242
243	if (maxsegsz == 0) {
244		return (EINVAL);
245	}
246
247	/* Return a NULL tag on failure */
248	*dmat = NULL;
249
250	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
251	    M_ZERO | M_NOWAIT);
252	if (newtag == NULL) {
253		CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
254		    __func__, newtag, 0, error);
255		return (ENOMEM);
256	}
257
258	newtag->parent = parent;
259	newtag->alignment = alignment;
260	newtag->boundary = boundary;
261	newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
262	newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
263	newtag->filter = filter;
264	newtag->filterarg = filterarg;
265	newtag->maxsize = maxsize;
266	newtag->nsegments = nsegments;
267	newtag->maxsegsz = maxsegsz;
268	newtag->flags = flags;
269	newtag->ref_count = 1; /* Count ourself */
270	newtag->map_count = 0;
271	if (lockfunc != NULL) {
272		newtag->lockfunc = lockfunc;
273		newtag->lockfuncarg = lockfuncarg;
274	} else {
275		newtag->lockfunc = dflt_lock;
276		newtag->lockfuncarg = NULL;
277	}
278
279	/* Take into account any restrictions imposed by our parent tag */
280	if (parent != NULL) {
281		newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
282		newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
283		if (newtag->boundary == 0)
284			newtag->boundary = parent->boundary;
285		else if (parent->boundary != 0)
286			newtag->boundary = MIN(parent->boundary,
287					       newtag->boundary);
288		if (newtag->filter == NULL) {
289			/*
290			 * Short circuit looking at our parent directly
291			 * since we have encapsulated all of its information
292			 */
293			newtag->filter = parent->filter;
294			newtag->filterarg = parent->filterarg;
295			newtag->parent = parent->parent;
296		}
297		if (newtag->parent != NULL)
298			atomic_add_int(&parent->ref_count, 1);
299		newtag->iommu = parent->iommu;
300		newtag->iommu_cookie = parent->iommu_cookie;
301	}
302
303	if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL)
304		newtag->flags |= BUS_DMA_COULD_BOUNCE;
305
306	if (newtag->alignment > 1)
307		newtag->flags |= BUS_DMA_COULD_BOUNCE;
308
309	if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
310	    (flags & BUS_DMA_ALLOCNOW) != 0) {
311		struct bounce_zone *bz;
312
313		/* Must bounce */
314
315		if ((error = alloc_bounce_zone(newtag)) != 0) {
316			free(newtag, M_DEVBUF);
317			return (error);
318		}
319		bz = newtag->bounce_zone;
320
321		if (ptoa(bz->total_bpages) < maxsize) {
322			int pages;
323
324			pages = atop(maxsize) - bz->total_bpages;
325
326			/* Add pages to our bounce pool */
327			if (alloc_bounce_pages(newtag, pages) < pages)
328				error = ENOMEM;
329		}
330		/* Performed initial allocation */
331		newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
332	}
333
334	if (error != 0) {
335		free(newtag, M_DEVBUF);
336	} else {
337		*dmat = newtag;
338	}
339	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
340	    __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
341	return (error);
342}
343
344void
345bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat)
346{
347
348	if (t == NULL || dmat == NULL)
349		return;
350
351	t->parent = dmat->parent;
352	t->alignment = dmat->alignment;
353	t->boundary = dmat->boundary;
354	t->lowaddr = dmat->lowaddr;
355	t->highaddr = dmat->highaddr;
356	t->maxsize = dmat->maxsize;
357	t->nsegments = dmat->nsegments;
358	t->maxsegsize = dmat->maxsegsz;
359	t->flags = dmat->flags;
360	t->lockfunc = dmat->lockfunc;
361	t->lockfuncarg = dmat->lockfuncarg;
362}
363
364int
365bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
366{
367
368	return (0);
369}
370
371int
372bus_dma_tag_destroy(bus_dma_tag_t dmat)
373{
374	bus_dma_tag_t dmat_copy __unused;
375	int error;
376
377	error = 0;
378	dmat_copy = dmat;
379
380	if (dmat != NULL) {
381		if (dmat->map_count != 0) {
382			error = EBUSY;
383			goto out;
384		}
385
386		while (dmat != NULL) {
387			bus_dma_tag_t parent;
388
389			parent = dmat->parent;
390			atomic_subtract_int(&dmat->ref_count, 1);
391			if (dmat->ref_count == 0) {
392				free(dmat, M_DEVBUF);
393				/*
394				 * Last reference count, so
395				 * release our reference
396				 * count on our parent.
397				 */
398				dmat = parent;
399			} else
400				dmat = NULL;
401		}
402	}
403out:
404	CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
405	return (error);
406}
407
408/*
409 * Allocate a handle for mapping from kva/uva/physical
410 * address space into bus device space.
411 */
412int
413bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
414{
415	int error;
416
417	error = 0;
418
419	*mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
420				     M_NOWAIT | M_ZERO);
421	if (*mapp == NULL) {
422		CTR3(KTR_BUSDMA, "%s: tag %p error %d",
423		    __func__, dmat, ENOMEM);
424		return (ENOMEM);
425	}
426
427	/*
428	 * Bouncing might be required if the driver asks for an active
429	 * exclusion region, a data alignment that is stricter than 1, and/or
430	 * an active address boundary.
431	 */
432	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
433		/* Must bounce */
434		struct bounce_zone *bz;
435		int maxpages;
436
437		if (dmat->bounce_zone == NULL) {
438			if ((error = alloc_bounce_zone(dmat)) != 0)
439				return (error);
440		}
441		bz = dmat->bounce_zone;
442
443		/* Initialize the new map */
444		STAILQ_INIT(&((*mapp)->bpages));
445
446		/*
447		 * Attempt to add pages to our pool on a per-instance
448		 * basis up to a sane limit.
449		 */
450		if (dmat->alignment > 1)
451			maxpages = MAX_BPAGES;
452		else
453			maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
454		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
455		 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
456			int pages;
457
458			pages = MAX(atop(dmat->maxsize), 1);
459			pages = MIN(maxpages - bz->total_bpages, pages);
460			pages = MAX(pages, 1);
461			if (alloc_bounce_pages(dmat, pages) < pages)
462				error = ENOMEM;
463
464			if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
465				if (error == 0)
466					dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
467			} else {
468				error = 0;
469			}
470		}
471		bz->map_count++;
472	}
473
474	(*mapp)->nsegs = 0;
475	(*mapp)->segments = (bus_dma_segment_t *)malloc(
476	    sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
477	    M_NOWAIT);
478	if ((*mapp)->segments == NULL) {
479		CTR3(KTR_BUSDMA, "%s: tag %p error %d",
480		    __func__, dmat, ENOMEM);
481		return (ENOMEM);
482	}
483
484	if (error == 0)
485		dmat->map_count++;
486	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
487	    __func__, dmat, dmat->flags, error);
488	return (error);
489}
490
491/*
492 * Destroy a handle for mapping from kva/uva/physical
493 * address space into bus device space.
494 */
495int
496bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
497{
498	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
499		if (STAILQ_FIRST(&map->bpages) != NULL) {
500			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
501			    __func__, dmat, EBUSY);
502			return (EBUSY);
503		}
504		if (dmat->bounce_zone)
505			dmat->bounce_zone->map_count--;
506	}
507	free(map->segments, M_DEVBUF);
508	free(map, M_DEVBUF);
509	dmat->map_count--;
510	CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
511	return (0);
512}
513
514/*
515 * Allocate a piece of memory that can be efficiently mapped into
516 * bus device space based on the constraints lited in the dma tag.
517 * A dmamap to for use with dmamap_load is also allocated.
518 */
519int
520bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
521		 bus_dmamap_t *mapp)
522{
523	vm_memattr_t attr;
524	int mflags;
525
526	if (flags & BUS_DMA_NOWAIT)
527		mflags = M_NOWAIT;
528	else
529		mflags = M_WAITOK;
530
531	bus_dmamap_create(dmat, flags, mapp);
532
533	if (flags & BUS_DMA_ZERO)
534		mflags |= M_ZERO;
535	if (flags & BUS_DMA_NOCACHE)
536		attr = VM_MEMATTR_UNCACHEABLE;
537	else
538		attr = VM_MEMATTR_DEFAULT;
539
540	/*
541	 * XXX:
542	 * (dmat->alignment <= dmat->maxsize) is just a quick hack; the exact
543	 * alignment guarantees of malloc need to be nailed down, and the
544	 * code below should be rewritten to take that into account.
545	 *
546	 * In the meantime, we'll warn the user if malloc gets it wrong.
547	 */
548	if ((dmat->maxsize <= PAGE_SIZE) &&
549	   (dmat->alignment <= dmat->maxsize) &&
550	    dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
551	    attr == VM_MEMATTR_DEFAULT) {
552		*vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
553	} else {
554		/*
555		 * XXX Use Contigmalloc until it is merged into this facility
556		 *     and handles multi-seg allocations.  Nobody is doing
557		 *     multi-seg allocations yet though.
558		 * XXX Certain AGP hardware does.
559		 */
560		*vaddr = (void *)kmem_alloc_contig(dmat->maxsize, mflags, 0ul,
561		    dmat->lowaddr, dmat->alignment ? dmat->alignment : 1ul,
562		    dmat->boundary, attr);
563		(*mapp)->contigalloc = 1;
564	}
565	if (*vaddr == NULL) {
566		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
567		    __func__, dmat, dmat->flags, ENOMEM);
568		return (ENOMEM);
569	} else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
570		printf("bus_dmamem_alloc failed to align memory properly.\n");
571	}
572	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
573	    __func__, dmat, dmat->flags, 0);
574	return (0);
575}
576
577/*
578 * Free a piece of memory and it's allociated dmamap, that was allocated
579 * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
580 */
581void
582bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
583{
584
585	if (!map->contigalloc)
586		free(vaddr, M_DEVBUF);
587	else
588		kmem_free((vm_offset_t)vaddr, dmat->maxsize);
589	bus_dmamap_destroy(dmat, map);
590	CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
591}
592
593static void
594_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
595    bus_size_t buflen, int flags)
596{
597	bus_addr_t curaddr;
598	bus_size_t sgsize;
599
600	if (map->pagesneeded == 0) {
601		CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
602		    "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
603		    dmat->boundary, dmat->alignment);
604		CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
605		/*
606		 * Count the number of bounce pages
607		 * needed in order to complete this transfer
608		 */
609		curaddr = buf;
610		while (buflen != 0) {
611			sgsize = MIN(buflen, dmat->maxsegsz);
612			if (run_filter(dmat, curaddr) != 0) {
613				sgsize = MIN(sgsize,
614				    PAGE_SIZE - (curaddr & PAGE_MASK));
615				map->pagesneeded++;
616			}
617			curaddr += sgsize;
618			buflen -= sgsize;
619		}
620		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
621	}
622}
623
624static void
625_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
626    void *buf, bus_size_t buflen, int flags)
627{
628        vm_offset_t vaddr;
629        vm_offset_t vendaddr;
630        bus_addr_t paddr;
631
632	if (map->pagesneeded == 0) {
633		CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
634		    "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
635		    dmat->boundary, dmat->alignment);
636		CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
637		/*
638		 * Count the number of bounce pages
639		 * needed in order to complete this transfer
640		 */
641		vaddr = (vm_offset_t)buf;
642		vendaddr = (vm_offset_t)buf + buflen;
643
644		while (vaddr < vendaddr) {
645			bus_size_t sg_len;
646
647			sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
648			if (pmap == kernel_pmap)
649				paddr = pmap_kextract(vaddr);
650			else
651				paddr = pmap_extract(pmap, vaddr);
652			if (run_filter(dmat, paddr) != 0) {
653				sg_len = roundup2(sg_len, dmat->alignment);
654				map->pagesneeded++;
655			}
656			vaddr += sg_len;
657		}
658		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
659	}
660}
661
662static int
663_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
664{
665
666	/* Reserve Necessary Bounce Pages */
667	mtx_lock(&bounce_lock);
668	if (flags & BUS_DMA_NOWAIT) {
669		if (reserve_bounce_pages(dmat, map, 0) != 0) {
670			mtx_unlock(&bounce_lock);
671			return (ENOMEM);
672		}
673	} else {
674		if (reserve_bounce_pages(dmat, map, 1) != 0) {
675			/* Queue us for resources */
676			STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
677			    map, links);
678			mtx_unlock(&bounce_lock);
679			return (EINPROGRESS);
680		}
681	}
682	mtx_unlock(&bounce_lock);
683
684	return (0);
685}
686
687/*
688 * Add a single contiguous physical range to the segment list.
689 */
690static int
691_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
692		   bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
693{
694	bus_addr_t baddr, bmask;
695	int seg;
696
697	/*
698	 * Make sure we don't cross any boundaries.
699	 */
700	bmask = ~(dmat->boundary - 1);
701	if (dmat->boundary > 0) {
702		baddr = (curaddr + dmat->boundary) & bmask;
703		if (sgsize > (baddr - curaddr))
704			sgsize = (baddr - curaddr);
705	}
706
707	/*
708	 * Insert chunk into a segment, coalescing with
709	 * previous segment if possible.
710	 */
711	seg = *segp;
712	if (seg == -1) {
713		seg = 0;
714		segs[seg].ds_addr = curaddr;
715		segs[seg].ds_len = sgsize;
716	} else {
717		if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
718		    (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
719		    (dmat->boundary == 0 ||
720		     (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
721			segs[seg].ds_len += sgsize;
722		else {
723			if (++seg >= dmat->nsegments)
724				return (0);
725			segs[seg].ds_addr = curaddr;
726			segs[seg].ds_len = sgsize;
727		}
728	}
729	*segp = seg;
730	return (sgsize);
731}
732
733/*
734 * Utility function to load a physical buffer.  segp contains
735 * the starting segment on entrace, and the ending segment on exit.
736 */
737int
738_bus_dmamap_load_phys(bus_dma_tag_t dmat,
739		      bus_dmamap_t map,
740		      vm_paddr_t buf, bus_size_t buflen,
741		      int flags,
742		      bus_dma_segment_t *segs,
743		      int *segp)
744{
745	bus_addr_t curaddr;
746	bus_size_t sgsize;
747	int error;
748
749	if (segs == NULL)
750		segs = map->segments;
751
752	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
753		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
754		if (map->pagesneeded != 0) {
755			error = _bus_dmamap_reserve_pages(dmat, map, flags);
756			if (error)
757				return (error);
758		}
759	}
760
761	while (buflen > 0) {
762		curaddr = buf;
763		sgsize = MIN(buflen, dmat->maxsegsz);
764		if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
765			sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
766			curaddr = add_bounce_page(dmat, map, 0, curaddr,
767			    sgsize);
768		}
769		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
770		    segp);
771		if (sgsize == 0)
772			break;
773		buf += sgsize;
774		buflen -= sgsize;
775	}
776
777	/*
778	 * Did we fit?
779	 */
780	return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
781}
782
783int
784_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
785    struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
786    bus_dma_segment_t *segs, int *segp)
787{
788
789	return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
790	    segs, segp));
791}
792
793/*
794 * Utility function to load a linear buffer.  segp contains
795 * the starting segment on entrance, and the ending segment on exit.
796 */
797int
798_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
799    			bus_dmamap_t map,
800			void *buf, bus_size_t buflen,
801			pmap_t pmap,
802			int flags,
803			bus_dma_segment_t *segs,
804			int *segp)
805{
806	bus_size_t sgsize;
807	bus_addr_t curaddr;
808	vm_offset_t kvaddr, vaddr;
809	int error;
810
811	if (segs == NULL)
812		segs = map->segments;
813
814	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
815		_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
816		if (map->pagesneeded != 0) {
817			error = _bus_dmamap_reserve_pages(dmat, map, flags);
818			if (error)
819				return (error);
820		}
821	}
822
823	vaddr = (vm_offset_t)buf;
824
825	while (buflen > 0) {
826		bus_size_t max_sgsize;
827
828		/*
829		 * Get the physical address for this segment.
830		 */
831		if (pmap == kernel_pmap) {
832			curaddr = pmap_kextract(vaddr);
833			kvaddr = vaddr;
834		} else {
835			curaddr = pmap_extract(pmap, vaddr);
836			kvaddr = 0;
837		}
838
839		/*
840		 * Compute the segment size, and adjust counts.
841		 */
842		max_sgsize = MIN(buflen, dmat->maxsegsz);
843		sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
844		if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
845			sgsize = roundup2(sgsize, dmat->alignment);
846			sgsize = MIN(sgsize, max_sgsize);
847			curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
848			    sgsize);
849		} else {
850			sgsize = MIN(sgsize, max_sgsize);
851		}
852
853		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
854		    segp);
855		if (sgsize == 0)
856			break;
857		vaddr += sgsize;
858		buflen -= sgsize;
859	}
860
861	/*
862	 * Did we fit?
863	 */
864	return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
865}
866
867void
868_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
869		    struct memdesc *mem, bus_dmamap_callback_t *callback,
870		    void *callback_arg)
871{
872
873	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
874		map->dmat = dmat;
875		map->mem = *mem;
876		map->callback = callback;
877		map->callback_arg = callback_arg;
878	}
879}
880
881bus_dma_segment_t *
882_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
883		     bus_dma_segment_t *segs, int nsegs, int error)
884{
885
886	map->nsegs = nsegs;
887	if (segs != NULL)
888		memcpy(map->segments, segs, map->nsegs*sizeof(segs[0]));
889	if (dmat->iommu != NULL)
890		IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs,
891		    dmat->lowaddr, dmat->highaddr, dmat->alignment,
892		    dmat->boundary, dmat->iommu_cookie);
893
894	if (segs != NULL)
895		memcpy(segs, map->segments, map->nsegs*sizeof(segs[0]));
896	else
897		segs = map->segments;
898
899	return (segs);
900}
901
902/*
903 * Release the mapping held by map.
904 */
905void
906bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
907{
908	struct bounce_page *bpage;
909
910	if (dmat->iommu) {
911		IOMMU_UNMAP(dmat->iommu, map->segments, map->nsegs, dmat->iommu_cookie);
912		map->nsegs = 0;
913	}
914
915	while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
916		STAILQ_REMOVE_HEAD(&map->bpages, links);
917		free_bounce_page(dmat, bpage);
918	}
919}
920
921void
922bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
923{
924	struct bounce_page *bpage;
925	vm_offset_t datavaddr, tempvaddr;
926
927	if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
928		/*
929		 * Handle data bouncing.  We might also
930		 * want to add support for invalidating
931		 * the caches on broken hardware
932		 */
933		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
934		    "performing bounce", __func__, dmat, dmat->flags, op);
935
936		if (op & BUS_DMASYNC_PREWRITE) {
937			while (bpage != NULL) {
938				tempvaddr = 0;
939				datavaddr = bpage->datavaddr;
940				if (datavaddr == 0) {
941					tempvaddr = pmap_quick_enter_page(
942					    bpage->datapage);
943					datavaddr = tempvaddr |
944					    bpage->dataoffs;
945				}
946
947				bcopy((void *)datavaddr,
948				    (void *)bpage->vaddr, bpage->datacount);
949
950				if (tempvaddr != 0)
951					pmap_quick_remove_page(tempvaddr);
952				bpage = STAILQ_NEXT(bpage, links);
953			}
954			dmat->bounce_zone->total_bounced++;
955		}
956
957		if (op & BUS_DMASYNC_POSTREAD) {
958			while (bpage != NULL) {
959				tempvaddr = 0;
960				datavaddr = bpage->datavaddr;
961				if (datavaddr == 0) {
962					tempvaddr = pmap_quick_enter_page(
963					    bpage->datapage);
964					datavaddr = tempvaddr |
965					    bpage->dataoffs;
966				}
967
968				bcopy((void *)bpage->vaddr,
969				    (void *)datavaddr, bpage->datacount);
970
971				if (tempvaddr != 0)
972					pmap_quick_remove_page(tempvaddr);
973				bpage = STAILQ_NEXT(bpage, links);
974			}
975			dmat->bounce_zone->total_bounced++;
976		}
977	}
978
979	powerpc_sync();
980}
981
982static void
983init_bounce_pages(void *dummy __unused)
984{
985
986	total_bpages = 0;
987	STAILQ_INIT(&bounce_zone_list);
988	STAILQ_INIT(&bounce_map_waitinglist);
989	STAILQ_INIT(&bounce_map_callbacklist);
990	mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
991}
992SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
993
994static struct sysctl_ctx_list *
995busdma_sysctl_tree(struct bounce_zone *bz)
996{
997	return (&bz->sysctl_tree);
998}
999
1000static struct sysctl_oid *
1001busdma_sysctl_tree_top(struct bounce_zone *bz)
1002{
1003	return (bz->sysctl_tree_top);
1004}
1005
1006static int
1007alloc_bounce_zone(bus_dma_tag_t dmat)
1008{
1009	struct bounce_zone *bz;
1010
1011	/* Check to see if we already have a suitable zone */
1012	STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1013		if ((dmat->alignment <= bz->alignment)
1014		 && (dmat->lowaddr >= bz->lowaddr)) {
1015			dmat->bounce_zone = bz;
1016			return (0);
1017		}
1018	}
1019
1020	if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
1021	    M_NOWAIT | M_ZERO)) == NULL)
1022		return (ENOMEM);
1023
1024	STAILQ_INIT(&bz->bounce_page_list);
1025	bz->free_bpages = 0;
1026	bz->reserved_bpages = 0;
1027	bz->active_bpages = 0;
1028	bz->lowaddr = dmat->lowaddr;
1029	bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1030	bz->map_count = 0;
1031	snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1032	busdma_zonecount++;
1033	snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1034	STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1035	dmat->bounce_zone = bz;
1036
1037	sysctl_ctx_init(&bz->sysctl_tree);
1038	bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1039	    SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1040	    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
1041	if (bz->sysctl_tree_top == NULL) {
1042		sysctl_ctx_free(&bz->sysctl_tree);
1043		return (0);	/* XXX error code? */
1044	}
1045
1046	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1047	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1048	    "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1049	    "Total bounce pages");
1050	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1051	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1052	    "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1053	    "Free bounce pages");
1054	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1055	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1056	    "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1057	    "Reserved bounce pages");
1058	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1059	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1060	    "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1061	    "Active bounce pages");
1062	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1063	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1064	    "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1065	    "Total bounce requests");
1066	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1067	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1068	    "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1069	    "Total bounce requests that were deferred");
1070	SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1071	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1072	    "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1073	SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz),
1074	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1075	    "alignment", CTLFLAG_RD, &bz->alignment, "");
1076
1077	return (0);
1078}
1079
1080static int
1081alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1082{
1083	struct bounce_zone *bz;
1084	int count;
1085
1086	bz = dmat->bounce_zone;
1087	count = 0;
1088	while (numpages > 0) {
1089		struct bounce_page *bpage;
1090
1091		bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1092						     M_NOWAIT | M_ZERO);
1093
1094		if (bpage == NULL)
1095			break;
1096		bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1097							 M_NOWAIT, 0ul,
1098							 bz->lowaddr,
1099							 PAGE_SIZE,
1100							 0);
1101		if (bpage->vaddr == 0) {
1102			free(bpage, M_DEVBUF);
1103			break;
1104		}
1105		bpage->busaddr = pmap_kextract(bpage->vaddr);
1106		mtx_lock(&bounce_lock);
1107		STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1108		total_bpages++;
1109		bz->total_bpages++;
1110		bz->free_bpages++;
1111		mtx_unlock(&bounce_lock);
1112		count++;
1113		numpages--;
1114	}
1115	return (count);
1116}
1117
1118static int
1119reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1120{
1121	struct bounce_zone *bz;
1122	int pages;
1123
1124	mtx_assert(&bounce_lock, MA_OWNED);
1125	bz = dmat->bounce_zone;
1126	pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1127	if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1128		return (map->pagesneeded - (map->pagesreserved + pages));
1129	bz->free_bpages -= pages;
1130	bz->reserved_bpages += pages;
1131	map->pagesreserved += pages;
1132	pages = map->pagesneeded - map->pagesreserved;
1133
1134	return (pages);
1135}
1136
1137static bus_addr_t
1138add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1139		bus_addr_t addr, bus_size_t size)
1140{
1141	struct bounce_zone *bz;
1142	struct bounce_page *bpage;
1143
1144	KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1145
1146	bz = dmat->bounce_zone;
1147	if (map->pagesneeded == 0)
1148		panic("add_bounce_page: map doesn't need any pages");
1149	map->pagesneeded--;
1150
1151	if (map->pagesreserved == 0)
1152		panic("add_bounce_page: map doesn't need any pages");
1153	map->pagesreserved--;
1154
1155	mtx_lock(&bounce_lock);
1156	bpage = STAILQ_FIRST(&bz->bounce_page_list);
1157	if (bpage == NULL)
1158		panic("add_bounce_page: free page list is empty");
1159
1160	STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1161	bz->reserved_bpages--;
1162	bz->active_bpages++;
1163	mtx_unlock(&bounce_lock);
1164
1165	if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1166		/* Page offset needs to be preserved. */
1167		bpage->vaddr |= addr & PAGE_MASK;
1168		bpage->busaddr |= addr & PAGE_MASK;
1169	}
1170	bpage->datavaddr = vaddr;
1171	bpage->datapage = PHYS_TO_VM_PAGE(addr);
1172	bpage->dataoffs = addr & PAGE_MASK;
1173	bpage->datacount = size;
1174	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1175	return (bpage->busaddr);
1176}
1177
1178static void
1179free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1180{
1181	struct bus_dmamap *map;
1182	struct bounce_zone *bz;
1183
1184	bz = dmat->bounce_zone;
1185	bpage->datavaddr = 0;
1186	bpage->datacount = 0;
1187	if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1188		/*
1189		 * Reset the bounce page to start at offset 0.  Other uses
1190		 * of this bounce page may need to store a full page of
1191		 * data and/or assume it starts on a page boundary.
1192		 */
1193		bpage->vaddr &= ~PAGE_MASK;
1194		bpage->busaddr &= ~PAGE_MASK;
1195	}
1196
1197	mtx_lock(&bounce_lock);
1198	STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1199	bz->free_bpages++;
1200	bz->active_bpages--;
1201	if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1202		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1203			STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1204			STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1205					   map, links);
1206			busdma_swi_pending = 1;
1207			bz->total_deferred++;
1208			swi_sched(vm_ih, 0);
1209		}
1210	}
1211	mtx_unlock(&bounce_lock);
1212}
1213
1214void
1215busdma_swi(void)
1216{
1217	bus_dma_tag_t dmat;
1218	struct bus_dmamap *map;
1219
1220	mtx_lock(&bounce_lock);
1221	while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1222		STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1223		mtx_unlock(&bounce_lock);
1224		dmat = map->dmat;
1225		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1226		bus_dmamap_load_mem(map->dmat, map, &map->mem,
1227				    map->callback, map->callback_arg,
1228				    BUS_DMA_WAITOK);
1229		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1230		mtx_lock(&bounce_lock);
1231	}
1232	mtx_unlock(&bounce_lock);
1233}
1234
1235int
1236bus_dma_tag_set_iommu(bus_dma_tag_t tag, device_t iommu, void *cookie)
1237{
1238	tag->iommu = iommu;
1239	tag->iommu_cookie = cookie;
1240
1241	return (0);
1242}
1243