34
35/*
36 * ARM bus dma support routines.
37 *
38 * XXX Things to investigate / fix some day...
39 * - What is the earliest that this API can be called? Could there be any
40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the
42 * bus_dmamap_load() function. This code has historically (and still does)
43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some
44 * error checking because some resource management calls would become WAITOK
45 * and thus "cannot fail."
46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag
47 * creation time, and the result stored in the tag.
48 * - It should be possible to take some shortcuts when mapping a buffer we know
49 * came from the uma(9) allocators based on what we know about such buffers
50 * (aligned, contiguous, etc).
51 * - The allocation of bounce pages could probably be cleaned up, then we could
52 * retire arm_remap_nocache().
53 */
54
55#define _ARM32_BUS_DMA_PRIVATE
56#include <sys/param.h>
57#include <sys/systm.h>
58#include <sys/malloc.h>
59#include <sys/bus.h>
60#include <sys/busdma_bufalloc.h>
61#include <sys/interrupt.h>
62#include <sys/lock.h>
63#include <sys/proc.h>
64#include <sys/mutex.h>
65#include <sys/mbuf.h>
66#include <sys/uio.h>
67#include <sys/ktr.h>
68#include <sys/kernel.h>
69#include <sys/sysctl.h>
70
71#include <vm/uma.h>
72#include <vm/vm.h>
73#include <vm/vm_extern.h>
74#include <vm/vm_kern.h>
75#include <vm/vm_page.h>
76#include <vm/vm_map.h>
77
78#include <machine/atomic.h>
79#include <machine/bus.h>
80#include <machine/cpufunc.h>
81#include <machine/md_var.h>
82
83#define MAX_BPAGES 64
84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
86
87struct bounce_zone;
88
89struct bus_dma_tag {
90 bus_dma_tag_t parent;
91 bus_size_t alignment;
92 bus_addr_t boundary;
93 bus_addr_t lowaddr;
94 bus_addr_t highaddr;
95 bus_dma_filter_t *filter;
96 void *filterarg;
97 bus_size_t maxsize;
98 u_int nsegments;
99 bus_size_t maxsegsz;
100 int flags;
101 int ref_count;
102 int map_count;
103 bus_dma_lock_t *lockfunc;
104 void *lockfuncarg;
105 /*
106 * DMA range for this tag. If the page doesn't fall within
107 * one of these ranges, an error is returned. The caller
108 * may then decide what to do with the transfer. If the
109 * range pointer is NULL, it is ignored.
110 */
111 struct arm32_dma_range *ranges;
112 int _nranges;
113 struct bounce_zone *bounce_zone;
114 /*
115 * Most tags need one or two segments, and can use the local tagsegs
116 * array. For tags with a larger limit, we'll allocate a bigger array
117 * on first use.
118 */
119 bus_dma_segment_t *segments;
120 bus_dma_segment_t tagsegs[2];
121};
122
123struct bounce_page {
124 vm_offset_t vaddr; /* kva of bounce buffer */
125 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */
126 bus_addr_t busaddr; /* Physical address */
127 vm_offset_t datavaddr; /* kva of client data */
128 bus_size_t datacount; /* client data count */
129 STAILQ_ENTRY(bounce_page) links;
130};
131
132int busdma_swi_pending;
133
134struct bounce_zone {
135 STAILQ_ENTRY(bounce_zone) links;
136 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
137 int total_bpages;
138 int free_bpages;
139 int reserved_bpages;
140 int active_bpages;
141 int total_bounced;
142 int total_deferred;
143 int map_count;
144 bus_size_t alignment;
145 bus_addr_t lowaddr;
146 char zoneid[8];
147 char lowaddrid[20];
148 struct sysctl_ctx_list sysctl_tree;
149 struct sysctl_oid *sysctl_tree_top;
150};
151
152static struct mtx bounce_lock;
153static int total_bpages;
154static int busdma_zonecount;
155static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
156
157static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
158SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
159 "Total bounce pages");
160
161#define DMAMAP_LINEAR 0x1
162#define DMAMAP_MBUF 0x2
163#define DMAMAP_UIO 0x4
164#define DMAMAP_CACHE_ALIGNED 0x10
165#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
166#define DMAMAP_COHERENT 0x8
167struct bus_dmamap {
168 struct bp_list bpages;
169 int pagesneeded;
170 int pagesreserved;
171 bus_dma_tag_t dmat;
172 int flags;
173 void *buffer;
174 int len;
175 STAILQ_ENTRY(bus_dmamap) links;
176 bus_dmamap_callback_t *callback;
177 void *callback_arg;
178
179};
180
181static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
182static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
183
184static struct mtx busdma_mtx;
185
186MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
187
188static void init_bounce_pages(void *dummy);
189static int alloc_bounce_zone(bus_dma_tag_t dmat);
190static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
191static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
192 int commit);
193static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
194 vm_offset_t vaddr, bus_size_t size);
195static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
196
197/* Default tag, as most drivers provide no parent tag. */
198bus_dma_tag_t arm_root_dma_tag;
199
200/*
201 * ----------------------------------------------------------------------------
202 * Begin block of code useful to transplant to other implementations.
203 */
204
205static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */
206
207static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
208static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
209
210/*
211 * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
212 * It'll need platform-specific changes if this code is copied.
213 */
214static int
215dmamap_ctor(void *mem, int size, void *arg, int flags)
216{
217 bus_dmamap_t map;
218 bus_dma_tag_t dmat;
219
220 map = (bus_dmamap_t)mem;
221 dmat = (bus_dma_tag_t)arg;
222
223 dmat->map_count++;
224
225 map->dmat = dmat;
226 map->flags = 0;
227 STAILQ_INIT(&map->bpages);
228
229 return (0);
230}
231
232/*
233 * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
234 * It may need platform-specific changes if this code is copied .
235 */
236static void
237dmamap_dtor(void *mem, int size, void *arg)
238{
239 bus_dmamap_t map;
240
241 map = (bus_dmamap_t)mem;
242
243 map->dmat->map_count--;
244}
245
246static void
247busdma_init(void *dummy)
248{
249
250 /* Create a cache of maps for bus_dmamap_create(). */
251 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
252 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
253
254 /* Create a cache of buffers in standard (cacheable) memory. */
255 standard_allocator = busdma_bufalloc_create("buffer",
256 arm_dcache_align, /* minimum_alignment */
257 NULL, /* uma_alloc func */
258 NULL, /* uma_free func */
259 0); /* uma_zcreate_flags */
260
261 /*
262 * Create a cache of buffers in uncacheable memory, to implement the
263 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag.
264 */
265 coherent_allocator = busdma_bufalloc_create("coherent",
266 arm_dcache_align, /* minimum_alignment */
267 busdma_bufalloc_alloc_uncacheable,
268 busdma_bufalloc_free_uncacheable,
269 0); /* uma_zcreate_flags */
270}
271
272/*
273 * This init historically used SI_SUB_VM, but now the init code requires
274 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by
275 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using
276 * SI_SUB_KMEM and SI_ORDER_THIRD.
277 */
278SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL);
279
280/*
281 * End block of code useful to transplant to other implementations.
282 * ----------------------------------------------------------------------------
283 */
284
285/*
286 * Return true if a match is made.
287 *
288 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
289 *
290 * If paddr is within the bounds of the dma tag then call the filter callback
291 * to check for a match, if there is no filter callback then assume a match.
292 */
293static int
294run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
295{
296 int retval;
297
298 retval = 0;
299
300 do {
301 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
302 || ((paddr & (dmat->alignment - 1)) != 0))
303 && (dmat->filter == NULL
304 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
305 retval = 1;
306
307 dmat = dmat->parent;
308 } while (retval == 0 && dmat != NULL);
309 return (retval);
310}
311
312/*
313 * This routine checks the exclusion zone constraints from a tag against the
314 * physical RAM available on the machine. If a tag specifies an exclusion zone
315 * but there's no RAM in that zone, then we avoid allocating resources to bounce
316 * a request, and we can use any memory allocator (as opposed to needing
317 * kmem_alloc_contig() just because it can allocate pages in an address range).
318 *
319 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the
320 * same value on 32-bit architectures) as their lowaddr constraint, and we can't
321 * possibly have RAM at an address higher than the highest address we can
322 * express, so we take a fast out.
323 */
324static __inline int
325_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
326{
327 int i;
328
329 if (lowaddr >= BUS_SPACE_MAXADDR)
330 return (0);
331
332 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
333 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
334 || (lowaddr < phys_avail[i] &&
335 highaddr > phys_avail[i]))
336 return (1);
337 }
338 return (0);
339}
340
341static __inline struct arm32_dma_range *
342_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
343 bus_addr_t curaddr)
344{
345 struct arm32_dma_range *dr;
346 int i;
347
348 for (i = 0, dr = ranges; i < nranges; i++, dr++) {
349 if (curaddr >= dr->dr_sysbase &&
350 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
351 return (dr);
352 }
353
354 return (NULL);
355}
356/*
357 * Convenience function for manipulating driver locks from busdma (during
358 * busdma_swi, for example). Drivers that don't provide their own locks
359 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
360 * non-mutex locking scheme don't have to use this at all.
361 */
362void
363busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
364{
365 struct mtx *dmtx;
366
367 dmtx = (struct mtx *)arg;
368 switch (op) {
369 case BUS_DMA_LOCK:
370 mtx_lock(dmtx);
371 break;
372 case BUS_DMA_UNLOCK:
373 mtx_unlock(dmtx);
374 break;
375 default:
376 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
377 }
378}
379
380/*
381 * dflt_lock should never get called. It gets put into the dma tag when
382 * lockfunc == NULL, which is only valid if the maps that are associated
383 * with the tag are meant to never be defered.
384 * XXX Should have a way to identify which driver is responsible here.
385 */
386static void
387dflt_lock(void *arg, bus_dma_lock_op_t op)
388{
389#ifdef INVARIANTS
390 panic("driver error: busdma dflt_lock called");
391#else
392 printf("DRIVER_ERROR: busdma dflt_lock called\n");
393#endif
394}
395
396/*
397 * Allocate a device specific dma_tag.
398 */
399#define SEG_NB 1024
400
401int
402bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
403 bus_addr_t boundary, bus_addr_t lowaddr,
404 bus_addr_t highaddr, bus_dma_filter_t *filter,
405 void *filterarg, bus_size_t maxsize, int nsegments,
406 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
407 void *lockfuncarg, bus_dma_tag_t *dmat)
408{
409 bus_dma_tag_t newtag;
410 int error = 0;
411 /* Return a NULL tag on failure */
412 *dmat = NULL;
413 if (!parent)
414 parent = arm_root_dma_tag;
415
416 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
417 if (newtag == NULL) {
418 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
419 __func__, newtag, 0, error);
420 return (ENOMEM);
421 }
422
423 newtag->parent = parent;
424 newtag->alignment = alignment ? alignment : 1;
425 newtag->boundary = boundary;
426 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
427 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
428 newtag->filter = filter;
429 newtag->filterarg = filterarg;
430 newtag->maxsize = maxsize;
431 newtag->nsegments = nsegments;
432 newtag->maxsegsz = maxsegsz;
433 newtag->flags = flags;
434 newtag->ref_count = 1; /* Count ourself */
435 newtag->map_count = 0;
436 newtag->ranges = bus_dma_get_range();
437 newtag->_nranges = bus_dma_get_range_nb();
438 if (lockfunc != NULL) {
439 newtag->lockfunc = lockfunc;
440 newtag->lockfuncarg = lockfuncarg;
441 } else {
442 newtag->lockfunc = dflt_lock;
443 newtag->lockfuncarg = NULL;
444 }
445 /*
446 * If all the segments we need fit into the local tagsegs array, set the
447 * pointer now. Otherwise NULL the pointer and an array of segments
448 * will be allocated later, on first use. We don't pre-allocate now
449 * because some tags exist just to pass contraints to children in the
450 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we
451 * sure don't want to try to allocate an array for that.
452 */
453 if (newtag->nsegments <= nitems(newtag->tagsegs))
454 newtag->segments = newtag->tagsegs;
455 else
456 newtag->segments = NULL;
457 /*
458 * Take into account any restrictions imposed by our parent tag
459 */
460 if (parent != NULL) {
461 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
462 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
463 if (newtag->boundary == 0)
464 newtag->boundary = parent->boundary;
465 else if (parent->boundary != 0)
466 newtag->boundary = MIN(parent->boundary,
467 newtag->boundary);
468 if ((newtag->filter != NULL) ||
469 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
470 newtag->flags |= BUS_DMA_COULD_BOUNCE;
471 if (newtag->filter == NULL) {
472 /*
473 * Short circuit looking at our parent directly
474 * since we have encapsulated all of its information
475 */
476 newtag->filter = parent->filter;
477 newtag->filterarg = parent->filterarg;
478 newtag->parent = parent->parent;
479 }
480 if (newtag->parent != NULL)
481 atomic_add_int(&parent->ref_count, 1);
482 }
483 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
484 || newtag->alignment > 1)
485 newtag->flags |= BUS_DMA_COULD_BOUNCE;
486
487 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
488 (flags & BUS_DMA_ALLOCNOW) != 0) {
489 struct bounce_zone *bz;
490
491 /* Must bounce */
492
493 if ((error = alloc_bounce_zone(newtag)) != 0) {
494 free(newtag, M_DEVBUF);
495 return (error);
496 }
497 bz = newtag->bounce_zone;
498
499 if (ptoa(bz->total_bpages) < maxsize) {
500 int pages;
501
502 pages = atop(maxsize) - bz->total_bpages;
503
504 /* Add pages to our bounce pool */
505 if (alloc_bounce_pages(newtag, pages) < pages)
506 error = ENOMEM;
507 }
508 /* Performed initial allocation */
509 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
510 } else
511 newtag->bounce_zone = NULL;
512 if (error != 0)
513 free(newtag, M_DEVBUF);
514 else
515 *dmat = newtag;
516 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
517 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
518
519 return (error);
520}
521
522int
523bus_dma_tag_destroy(bus_dma_tag_t dmat)
524{
525#ifdef KTR
526 bus_dma_tag_t dmat_copy = dmat;
527#endif
528
529 if (dmat != NULL) {
530
531 if (dmat->map_count != 0)
532 return (EBUSY);
533
534 while (dmat != NULL) {
535 bus_dma_tag_t parent;
536
537 parent = dmat->parent;
538 atomic_subtract_int(&dmat->ref_count, 1);
539 if (dmat->ref_count == 0) {
540 if (dmat->segments != NULL &&
541 dmat->segments != dmat->tagsegs)
542 free(dmat->segments, M_DEVBUF);
543 free(dmat, M_DEVBUF);
544 /*
545 * Last reference count, so
546 * release our reference
547 * count on our parent.
548 */
549 dmat = parent;
550 } else
551 dmat = NULL;
552 }
553 }
554 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
555
556 return (0);
557}
558
559#include <sys/kdb.h>
560/*
561 * Allocate a handle for mapping from kva/uva/physical
562 * address space into bus device space.
563 */
564int
565bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
566{
567 bus_dmamap_t map;
568 int error = 0;
569
| 34
35/*
36 * ARM bus dma support routines.
37 *
38 * XXX Things to investigate / fix some day...
39 * - What is the earliest that this API can be called? Could there be any
40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the
42 * bus_dmamap_load() function. This code has historically (and still does)
43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some
44 * error checking because some resource management calls would become WAITOK
45 * and thus "cannot fail."
46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag
47 * creation time, and the result stored in the tag.
48 * - It should be possible to take some shortcuts when mapping a buffer we know
49 * came from the uma(9) allocators based on what we know about such buffers
50 * (aligned, contiguous, etc).
51 * - The allocation of bounce pages could probably be cleaned up, then we could
52 * retire arm_remap_nocache().
53 */
54
55#define _ARM32_BUS_DMA_PRIVATE
56#include <sys/param.h>
57#include <sys/systm.h>
58#include <sys/malloc.h>
59#include <sys/bus.h>
60#include <sys/busdma_bufalloc.h>
61#include <sys/interrupt.h>
62#include <sys/lock.h>
63#include <sys/proc.h>
64#include <sys/mutex.h>
65#include <sys/mbuf.h>
66#include <sys/uio.h>
67#include <sys/ktr.h>
68#include <sys/kernel.h>
69#include <sys/sysctl.h>
70
71#include <vm/uma.h>
72#include <vm/vm.h>
73#include <vm/vm_extern.h>
74#include <vm/vm_kern.h>
75#include <vm/vm_page.h>
76#include <vm/vm_map.h>
77
78#include <machine/atomic.h>
79#include <machine/bus.h>
80#include <machine/cpufunc.h>
81#include <machine/md_var.h>
82
83#define MAX_BPAGES 64
84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
86
87struct bounce_zone;
88
89struct bus_dma_tag {
90 bus_dma_tag_t parent;
91 bus_size_t alignment;
92 bus_addr_t boundary;
93 bus_addr_t lowaddr;
94 bus_addr_t highaddr;
95 bus_dma_filter_t *filter;
96 void *filterarg;
97 bus_size_t maxsize;
98 u_int nsegments;
99 bus_size_t maxsegsz;
100 int flags;
101 int ref_count;
102 int map_count;
103 bus_dma_lock_t *lockfunc;
104 void *lockfuncarg;
105 /*
106 * DMA range for this tag. If the page doesn't fall within
107 * one of these ranges, an error is returned. The caller
108 * may then decide what to do with the transfer. If the
109 * range pointer is NULL, it is ignored.
110 */
111 struct arm32_dma_range *ranges;
112 int _nranges;
113 struct bounce_zone *bounce_zone;
114 /*
115 * Most tags need one or two segments, and can use the local tagsegs
116 * array. For tags with a larger limit, we'll allocate a bigger array
117 * on first use.
118 */
119 bus_dma_segment_t *segments;
120 bus_dma_segment_t tagsegs[2];
121};
122
123struct bounce_page {
124 vm_offset_t vaddr; /* kva of bounce buffer */
125 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */
126 bus_addr_t busaddr; /* Physical address */
127 vm_offset_t datavaddr; /* kva of client data */
128 bus_size_t datacount; /* client data count */
129 STAILQ_ENTRY(bounce_page) links;
130};
131
132int busdma_swi_pending;
133
134struct bounce_zone {
135 STAILQ_ENTRY(bounce_zone) links;
136 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
137 int total_bpages;
138 int free_bpages;
139 int reserved_bpages;
140 int active_bpages;
141 int total_bounced;
142 int total_deferred;
143 int map_count;
144 bus_size_t alignment;
145 bus_addr_t lowaddr;
146 char zoneid[8];
147 char lowaddrid[20];
148 struct sysctl_ctx_list sysctl_tree;
149 struct sysctl_oid *sysctl_tree_top;
150};
151
152static struct mtx bounce_lock;
153static int total_bpages;
154static int busdma_zonecount;
155static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
156
157static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
158SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
159 "Total bounce pages");
160
161#define DMAMAP_LINEAR 0x1
162#define DMAMAP_MBUF 0x2
163#define DMAMAP_UIO 0x4
164#define DMAMAP_CACHE_ALIGNED 0x10
165#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
166#define DMAMAP_COHERENT 0x8
167struct bus_dmamap {
168 struct bp_list bpages;
169 int pagesneeded;
170 int pagesreserved;
171 bus_dma_tag_t dmat;
172 int flags;
173 void *buffer;
174 int len;
175 STAILQ_ENTRY(bus_dmamap) links;
176 bus_dmamap_callback_t *callback;
177 void *callback_arg;
178
179};
180
181static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
182static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
183
184static struct mtx busdma_mtx;
185
186MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
187
188static void init_bounce_pages(void *dummy);
189static int alloc_bounce_zone(bus_dma_tag_t dmat);
190static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
191static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
192 int commit);
193static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
194 vm_offset_t vaddr, bus_size_t size);
195static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
196
197/* Default tag, as most drivers provide no parent tag. */
198bus_dma_tag_t arm_root_dma_tag;
199
200/*
201 * ----------------------------------------------------------------------------
202 * Begin block of code useful to transplant to other implementations.
203 */
204
205static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */
206
207static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
208static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
209
210/*
211 * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
212 * It'll need platform-specific changes if this code is copied.
213 */
214static int
215dmamap_ctor(void *mem, int size, void *arg, int flags)
216{
217 bus_dmamap_t map;
218 bus_dma_tag_t dmat;
219
220 map = (bus_dmamap_t)mem;
221 dmat = (bus_dma_tag_t)arg;
222
223 dmat->map_count++;
224
225 map->dmat = dmat;
226 map->flags = 0;
227 STAILQ_INIT(&map->bpages);
228
229 return (0);
230}
231
232/*
233 * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
234 * It may need platform-specific changes if this code is copied .
235 */
236static void
237dmamap_dtor(void *mem, int size, void *arg)
238{
239 bus_dmamap_t map;
240
241 map = (bus_dmamap_t)mem;
242
243 map->dmat->map_count--;
244}
245
246static void
247busdma_init(void *dummy)
248{
249
250 /* Create a cache of maps for bus_dmamap_create(). */
251 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
252 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
253
254 /* Create a cache of buffers in standard (cacheable) memory. */
255 standard_allocator = busdma_bufalloc_create("buffer",
256 arm_dcache_align, /* minimum_alignment */
257 NULL, /* uma_alloc func */
258 NULL, /* uma_free func */
259 0); /* uma_zcreate_flags */
260
261 /*
262 * Create a cache of buffers in uncacheable memory, to implement the
263 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag.
264 */
265 coherent_allocator = busdma_bufalloc_create("coherent",
266 arm_dcache_align, /* minimum_alignment */
267 busdma_bufalloc_alloc_uncacheable,
268 busdma_bufalloc_free_uncacheable,
269 0); /* uma_zcreate_flags */
270}
271
272/*
273 * This init historically used SI_SUB_VM, but now the init code requires
274 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by
275 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using
276 * SI_SUB_KMEM and SI_ORDER_THIRD.
277 */
278SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL);
279
280/*
281 * End block of code useful to transplant to other implementations.
282 * ----------------------------------------------------------------------------
283 */
284
285/*
286 * Return true if a match is made.
287 *
288 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
289 *
290 * If paddr is within the bounds of the dma tag then call the filter callback
291 * to check for a match, if there is no filter callback then assume a match.
292 */
293static int
294run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
295{
296 int retval;
297
298 retval = 0;
299
300 do {
301 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
302 || ((paddr & (dmat->alignment - 1)) != 0))
303 && (dmat->filter == NULL
304 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
305 retval = 1;
306
307 dmat = dmat->parent;
308 } while (retval == 0 && dmat != NULL);
309 return (retval);
310}
311
312/*
313 * This routine checks the exclusion zone constraints from a tag against the
314 * physical RAM available on the machine. If a tag specifies an exclusion zone
315 * but there's no RAM in that zone, then we avoid allocating resources to bounce
316 * a request, and we can use any memory allocator (as opposed to needing
317 * kmem_alloc_contig() just because it can allocate pages in an address range).
318 *
319 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the
320 * same value on 32-bit architectures) as their lowaddr constraint, and we can't
321 * possibly have RAM at an address higher than the highest address we can
322 * express, so we take a fast out.
323 */
324static __inline int
325_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
326{
327 int i;
328
329 if (lowaddr >= BUS_SPACE_MAXADDR)
330 return (0);
331
332 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
333 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
334 || (lowaddr < phys_avail[i] &&
335 highaddr > phys_avail[i]))
336 return (1);
337 }
338 return (0);
339}
340
341static __inline struct arm32_dma_range *
342_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
343 bus_addr_t curaddr)
344{
345 struct arm32_dma_range *dr;
346 int i;
347
348 for (i = 0, dr = ranges; i < nranges; i++, dr++) {
349 if (curaddr >= dr->dr_sysbase &&
350 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
351 return (dr);
352 }
353
354 return (NULL);
355}
356/*
357 * Convenience function for manipulating driver locks from busdma (during
358 * busdma_swi, for example). Drivers that don't provide their own locks
359 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
360 * non-mutex locking scheme don't have to use this at all.
361 */
362void
363busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
364{
365 struct mtx *dmtx;
366
367 dmtx = (struct mtx *)arg;
368 switch (op) {
369 case BUS_DMA_LOCK:
370 mtx_lock(dmtx);
371 break;
372 case BUS_DMA_UNLOCK:
373 mtx_unlock(dmtx);
374 break;
375 default:
376 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
377 }
378}
379
380/*
381 * dflt_lock should never get called. It gets put into the dma tag when
382 * lockfunc == NULL, which is only valid if the maps that are associated
383 * with the tag are meant to never be defered.
384 * XXX Should have a way to identify which driver is responsible here.
385 */
386static void
387dflt_lock(void *arg, bus_dma_lock_op_t op)
388{
389#ifdef INVARIANTS
390 panic("driver error: busdma dflt_lock called");
391#else
392 printf("DRIVER_ERROR: busdma dflt_lock called\n");
393#endif
394}
395
396/*
397 * Allocate a device specific dma_tag.
398 */
399#define SEG_NB 1024
400
401int
402bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
403 bus_addr_t boundary, bus_addr_t lowaddr,
404 bus_addr_t highaddr, bus_dma_filter_t *filter,
405 void *filterarg, bus_size_t maxsize, int nsegments,
406 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
407 void *lockfuncarg, bus_dma_tag_t *dmat)
408{
409 bus_dma_tag_t newtag;
410 int error = 0;
411 /* Return a NULL tag on failure */
412 *dmat = NULL;
413 if (!parent)
414 parent = arm_root_dma_tag;
415
416 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
417 if (newtag == NULL) {
418 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
419 __func__, newtag, 0, error);
420 return (ENOMEM);
421 }
422
423 newtag->parent = parent;
424 newtag->alignment = alignment ? alignment : 1;
425 newtag->boundary = boundary;
426 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
427 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
428 newtag->filter = filter;
429 newtag->filterarg = filterarg;
430 newtag->maxsize = maxsize;
431 newtag->nsegments = nsegments;
432 newtag->maxsegsz = maxsegsz;
433 newtag->flags = flags;
434 newtag->ref_count = 1; /* Count ourself */
435 newtag->map_count = 0;
436 newtag->ranges = bus_dma_get_range();
437 newtag->_nranges = bus_dma_get_range_nb();
438 if (lockfunc != NULL) {
439 newtag->lockfunc = lockfunc;
440 newtag->lockfuncarg = lockfuncarg;
441 } else {
442 newtag->lockfunc = dflt_lock;
443 newtag->lockfuncarg = NULL;
444 }
445 /*
446 * If all the segments we need fit into the local tagsegs array, set the
447 * pointer now. Otherwise NULL the pointer and an array of segments
448 * will be allocated later, on first use. We don't pre-allocate now
449 * because some tags exist just to pass contraints to children in the
450 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we
451 * sure don't want to try to allocate an array for that.
452 */
453 if (newtag->nsegments <= nitems(newtag->tagsegs))
454 newtag->segments = newtag->tagsegs;
455 else
456 newtag->segments = NULL;
457 /*
458 * Take into account any restrictions imposed by our parent tag
459 */
460 if (parent != NULL) {
461 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
462 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
463 if (newtag->boundary == 0)
464 newtag->boundary = parent->boundary;
465 else if (parent->boundary != 0)
466 newtag->boundary = MIN(parent->boundary,
467 newtag->boundary);
468 if ((newtag->filter != NULL) ||
469 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
470 newtag->flags |= BUS_DMA_COULD_BOUNCE;
471 if (newtag->filter == NULL) {
472 /*
473 * Short circuit looking at our parent directly
474 * since we have encapsulated all of its information
475 */
476 newtag->filter = parent->filter;
477 newtag->filterarg = parent->filterarg;
478 newtag->parent = parent->parent;
479 }
480 if (newtag->parent != NULL)
481 atomic_add_int(&parent->ref_count, 1);
482 }
483 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
484 || newtag->alignment > 1)
485 newtag->flags |= BUS_DMA_COULD_BOUNCE;
486
487 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
488 (flags & BUS_DMA_ALLOCNOW) != 0) {
489 struct bounce_zone *bz;
490
491 /* Must bounce */
492
493 if ((error = alloc_bounce_zone(newtag)) != 0) {
494 free(newtag, M_DEVBUF);
495 return (error);
496 }
497 bz = newtag->bounce_zone;
498
499 if (ptoa(bz->total_bpages) < maxsize) {
500 int pages;
501
502 pages = atop(maxsize) - bz->total_bpages;
503
504 /* Add pages to our bounce pool */
505 if (alloc_bounce_pages(newtag, pages) < pages)
506 error = ENOMEM;
507 }
508 /* Performed initial allocation */
509 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
510 } else
511 newtag->bounce_zone = NULL;
512 if (error != 0)
513 free(newtag, M_DEVBUF);
514 else
515 *dmat = newtag;
516 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
517 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
518
519 return (error);
520}
521
522int
523bus_dma_tag_destroy(bus_dma_tag_t dmat)
524{
525#ifdef KTR
526 bus_dma_tag_t dmat_copy = dmat;
527#endif
528
529 if (dmat != NULL) {
530
531 if (dmat->map_count != 0)
532 return (EBUSY);
533
534 while (dmat != NULL) {
535 bus_dma_tag_t parent;
536
537 parent = dmat->parent;
538 atomic_subtract_int(&dmat->ref_count, 1);
539 if (dmat->ref_count == 0) {
540 if (dmat->segments != NULL &&
541 dmat->segments != dmat->tagsegs)
542 free(dmat->segments, M_DEVBUF);
543 free(dmat, M_DEVBUF);
544 /*
545 * Last reference count, so
546 * release our reference
547 * count on our parent.
548 */
549 dmat = parent;
550 } else
551 dmat = NULL;
552 }
553 }
554 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
555
556 return (0);
557}
558
559#include <sys/kdb.h>
560/*
561 * Allocate a handle for mapping from kva/uva/physical
562 * address space into bus device space.
563 */
564int
565bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
566{
567 bus_dmamap_t map;
568 int error = 0;
569
|
580
581 /*
582 * Bouncing might be required if the driver asks for an active
583 * exclusion region, a data alignment that is stricter than 1, and/or
584 * an active address boundary.
585 */
586 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
587
588 /* Must bounce */
589 struct bounce_zone *bz;
590 int maxpages;
591
592 if (dmat->bounce_zone == NULL) {
593 if ((error = alloc_bounce_zone(dmat)) != 0) {
594 uma_zfree(dmamap_zone, map);
595 *mapp = NULL;
596 return (error);
597 }
598 }
599 bz = dmat->bounce_zone;
600
601 /* Initialize the new map */
602 STAILQ_INIT(&((*mapp)->bpages));
603
604 /*
605 * Attempt to add pages to our pool on a per-instance
606 * basis up to a sane limit.
607 */
608 maxpages = MAX_BPAGES;
609 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
610 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
611 int pages;
612
613 pages = MAX(atop(dmat->maxsize), 1);
614 pages = MIN(maxpages - bz->total_bpages, pages);
615 pages = MAX(pages, 1);
616 if (alloc_bounce_pages(dmat, pages) < pages)
617 error = ENOMEM;
618
619 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
620 if (error == 0)
621 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
622 } else {
623 error = 0;
624 }
625 }
626 bz->map_count++;
627 }
628 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
629 __func__, dmat, dmat->flags, error);
630
631 return (0);
632}
633
634/*
635 * Destroy a handle for mapping from kva/uva/physical
636 * address space into bus device space.
637 */
638int
639bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
640{
641
642 if (STAILQ_FIRST(&map->bpages) != NULL) {
643 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
644 __func__, dmat, EBUSY);
645 return (EBUSY);
646 }
647 uma_zfree(dmamap_zone, map);
648 if (dmat->bounce_zone)
649 dmat->bounce_zone->map_count--;
650 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
651 return (0);
652}
653
654/*
655 * Allocate a piece of memory that can be efficiently mapped into bus device
656 * space based on the constraints listed in the dma tag. Returns a pointer to
657 * the allocated memory, and a pointer to an associated bus_dmamap.
658 */
659int
660bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags,
661 bus_dmamap_t *mapp)
662{
663 void * vaddr;
664 struct busdma_bufzone *bufzone;
665 busdma_bufalloc_t ba;
666 bus_dmamap_t map;
667 int mflags;
668 vm_memattr_t memattr;
669
670 if (flags & BUS_DMA_NOWAIT)
671 mflags = M_NOWAIT;
672 else
673 mflags = M_WAITOK;
674
675 /*
676 * If the tag's segments haven't been allocated yet we need to do it
677 * now, because we can't sleep for resources at map load time.
678 */
679 if (dmat->segments == NULL)
680 dmat->segments = malloc(dmat->nsegments *
681 sizeof(*dmat->segments), M_DEVBUF, mflags);
682
683 map = uma_zalloc_arg(dmamap_zone, dmat, mflags);
684 if (map == NULL)
685 return (ENOMEM);
686
687 if (flags & BUS_DMA_COHERENT) {
688 memattr = VM_MEMATTR_UNCACHEABLE;
689 ba = coherent_allocator;
690 map->flags |= DMAMAP_COHERENT;
691 } else {
692 memattr = VM_MEMATTR_DEFAULT;
693 ba = standard_allocator;
694 }
695 /* All buffers we allocate are cache-aligned. */
696 map->flags |= DMAMAP_CACHE_ALIGNED;
697
698 if (flags & BUS_DMA_ZERO)
699 mflags |= M_ZERO;
700
701 /*
702 * Try to find a bufzone in the allocator that holds a cache of buffers
703 * of the right size for this request. If the buffer is too big to be
704 * held in the allocator cache, this returns NULL.
705 */
706 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
707
708 /*
709 * Allocate the buffer from the uma(9) allocator if...
710 * - It's small enough to be in the allocator (bufzone not NULL).
711 * - The alignment constraint isn't larger than the allocation size
712 * (the allocator aligns buffers to their size boundaries).
713 * - There's no need to handle lowaddr/highaddr exclusion zones.
714 * else allocate non-contiguous pages if...
715 * - The page count that could get allocated doesn't exceed nsegments.
716 * - The alignment constraint isn't larger than a page boundary.
717 * - There are no boundary-crossing constraints.
718 * else allocate a block of contiguous pages because one or more of the
719 * constraints is something that only the contig allocator can fulfill.
720 */
721 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
722 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
723 vaddr = uma_zalloc(bufzone->umazone, mflags);
724 } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
725 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) {
726 vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize,
727 mflags, 0, dmat->lowaddr, memattr);
728 } else {
729 vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize,
730 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
731 memattr);
732 }
733
734 if (vaddr == NULL) {
735 uma_zfree(dmamap_zone, map);
736 map = NULL;
737 }
738
739 *vaddrp = vaddr;
740 *mapp = map;
741
742 return (vaddr == NULL ? ENOMEM : 0);
743}
744
745/*
746 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with
747 * its associated map.
748 */
749void
750bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
751{
752 struct busdma_bufzone *bufzone;
753 busdma_bufalloc_t ba;
754
755 if (map->flags & DMAMAP_COHERENT)
756 ba = coherent_allocator;
757 else
758 ba = standard_allocator;
759 uma_zfree(dmamap_zone, map);
760
761 /* Be careful not to access map from here on. */
762
763 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
764
765 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
766 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
767 uma_zfree(bufzone->umazone, vaddr);
768 else
769 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize);
770}
771
772static int
773_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
774 void *buf, bus_size_t buflen, int flags)
775{
776 vm_offset_t vaddr;
777 vm_offset_t vendaddr;
778 bus_addr_t paddr;
779
780 if ((map->pagesneeded == 0)) {
781 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
782 dmat->lowaddr, dmat->boundary, dmat->alignment);
783 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
784 map, map->pagesneeded);
785 /*
786 * Count the number of bounce pages
787 * needed in order to complete this transfer
788 */
789 vaddr = trunc_page((vm_offset_t)buf);
790 vendaddr = (vm_offset_t)buf + buflen;
791
792 while (vaddr < vendaddr) {
793 if (__predict_true(pmap == pmap_kernel()))
794 paddr = pmap_kextract(vaddr);
795 else
796 paddr = pmap_extract(pmap, vaddr);
797 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
798 run_filter(dmat, paddr) != 0)
799 map->pagesneeded++;
800 vaddr += PAGE_SIZE;
801 }
802 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
803 }
804
805 /* Reserve Necessary Bounce Pages */
806 if (map->pagesneeded != 0) {
807 mtx_lock(&bounce_lock);
808 if (flags & BUS_DMA_NOWAIT) {
809 if (reserve_bounce_pages(dmat, map, 0) != 0) {
810 mtx_unlock(&bounce_lock);
811 return (ENOMEM);
812 }
813 } else {
814 if (reserve_bounce_pages(dmat, map, 1) != 0) {
815 /* Queue us for resources */
816 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
817 map, links);
818 mtx_unlock(&bounce_lock);
819 return (EINPROGRESS);
820 }
821 }
822 mtx_unlock(&bounce_lock);
823 }
824
825 return (0);
826}
827
828/*
829 * Utility function to load a linear buffer. lastaddrp holds state
830 * between invocations (for multiple-buffer loads). segp contains
831 * the starting segment on entrance, and the ending segment on exit.
832 * first indicates if this is the first invocation of this function.
833 */
834static __inline int
835bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs,
836 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap,
837 int flags, vm_offset_t *lastaddrp, int *segp)
838{
839 bus_size_t sgsize;
840 bus_addr_t curaddr, lastaddr, baddr, bmask;
841 vm_offset_t vaddr = (vm_offset_t)buf;
842 int seg;
843 int error = 0;
844 pd_entry_t *pde;
845 pt_entry_t pte;
846 pt_entry_t *ptep;
847
848 lastaddr = *lastaddrp;
849 bmask = ~(dmat->boundary - 1);
850
851 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
852 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen,
853 flags);
854 if (error)
855 return (error);
856 }
857 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
858 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
859
860 for (seg = *segp; buflen > 0 ; ) {
861 /*
862 * Get the physical address for this segment.
863 *
864 * XXX Don't support checking for coherent mappings
865 * XXX in user address space.
866 */
867 if (__predict_true(pmap == pmap_kernel())) {
868 if (pmap_get_pde_pte(pmap, vaddr, &pde, &ptep) == FALSE)
869 return (EFAULT);
870
871 if (__predict_false(pmap_pde_section(pde))) {
872 if (*pde & L1_S_SUPERSEC)
873 curaddr = (*pde & L1_SUP_FRAME) |
874 (vaddr & L1_SUP_OFFSET);
875 else
876 curaddr = (*pde & L1_S_FRAME) |
877 (vaddr & L1_S_OFFSET);
878 } else {
879 pte = *ptep;
880 KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV,
881 ("INV type"));
882 if (__predict_false((pte & L2_TYPE_MASK)
883 == L2_TYPE_L)) {
884 curaddr = (pte & L2_L_FRAME) |
885 (vaddr & L2_L_OFFSET);
886 } else {
887 curaddr = (pte & L2_S_FRAME) |
888 (vaddr & L2_S_OFFSET);
889 }
890 }
891 } else {
892 curaddr = pmap_extract(pmap, vaddr);
893 map->flags &= ~DMAMAP_COHERENT;
894 }
895
896 /*
897 * Compute the segment size, and adjust counts.
898 */
899 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
900 if (sgsize > dmat->maxsegsz)
901 sgsize = dmat->maxsegsz;
902 if (buflen < sgsize)
903 sgsize = buflen;
904
905 /*
906 * Make sure we don't cross any boundaries.
907 */
908 if (dmat->boundary > 0) {
909 baddr = (curaddr + dmat->boundary) & bmask;
910 if (sgsize > (baddr - curaddr))
911 sgsize = (baddr - curaddr);
912 }
913 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
914 map->pagesneeded != 0 && run_filter(dmat, curaddr))
915 curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
916
917 if (dmat->ranges) {
918 struct arm32_dma_range *dr;
919
920 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
921 curaddr);
922 if (dr == NULL)
923 return (EINVAL);
924 /*
925 * In a valid DMA range. Translate the physical
926 * memory address to an address in the DMA window.
927 */
928 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
929
930 }
931
932 /*
933 * Insert chunk into a segment, coalescing with
934 * the previous segment if possible.
935 */
936 if (seg >= 0 && curaddr == lastaddr &&
937 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
938 (dmat->boundary == 0 ||
939 (segs[seg].ds_addr & bmask) ==
940 (curaddr & bmask))) {
941 segs[seg].ds_len += sgsize;
942 goto segdone;
943 } else {
944 if (++seg >= dmat->nsegments)
945 break;
946 segs[seg].ds_addr = curaddr;
947 segs[seg].ds_len = sgsize;
948 }
949 if (error)
950 break;
951segdone:
952 lastaddr = curaddr + sgsize;
953 vaddr += sgsize;
954 buflen -= sgsize;
955 }
956
957 *segp = seg;
958 *lastaddrp = lastaddr;
959
960 /*
961 * Did we fit?
962 */
963 if (buflen != 0)
964 error = EFBIG; /* XXX better return value here? */
965 return (error);
966}
967
968/*
969 * Map the buffer buf into bus space using the dmamap map.
970 */
971int
972bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
973 bus_size_t buflen, bus_dmamap_callback_t *callback,
974 void *callback_arg, int flags)
975{
976 vm_offset_t lastaddr = 0;
977 int error, nsegs = -1;
978
979 KASSERT(dmat != NULL, ("dmatag is NULL"));
980 KASSERT(map != NULL, ("dmamap is NULL"));
981 map->callback = callback;
982 map->callback_arg = callback_arg;
983 map->flags &= ~DMAMAP_TYPE_MASK;
984 map->flags |= DMAMAP_LINEAR;
985 map->buffer = buf;
986 map->len = buflen;
987 error = bus_dmamap_load_buffer(dmat,
988 dmat->segments, map, buf, buflen, kernel_pmap,
989 flags, &lastaddr, &nsegs);
990 if (error == EINPROGRESS)
991 return (error);
992 if (error)
993 (*callback)(callback_arg, NULL, 0, error);
994 else
995 (*callback)(callback_arg, dmat->segments, nsegs + 1, error);
996
997 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
998 __func__, dmat, dmat->flags, nsegs + 1, error);
999
1000 return (error);
1001}
1002
1003/*
1004 * Like bus_dmamap_load(), but for mbufs.
1005 *
1006 * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs.
1007 *
1008 * We know that the way the system allocates and uses mbufs implies that we can
1009 * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line
1010 * flushes. Even though the flush may reference the data area within the mbuf
1011 * that isn't aligned to a cache line, we know the overall mbuf itself is
1012 * properly aligned, and we know that the CPU will not touch the header fields
1013 * before the data area while the DMA is in progress.
1014 */
1015int
1016bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
1017 bus_dmamap_callback2_t *callback, void *callback_arg,
1018 int flags)
1019{
1020 int nsegs = -1, error = 0;
1021
1022 M_ASSERTPKTHDR(m0);
1023
1024 flags |= BUS_DMA_NOWAIT;
1025 map->flags &= ~DMAMAP_TYPE_MASK;
1026 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
1027 map->buffer = m0;
1028 map->len = 0;
1029 if (m0->m_pkthdr.len <= dmat->maxsize) {
1030 vm_offset_t lastaddr = 0;
1031 struct mbuf *m;
1032
1033 for (m = m0; m != NULL && error == 0; m = m->m_next) {
1034 if (m->m_len > 0) {
1035 error = bus_dmamap_load_buffer(dmat,
1036 dmat->segments, map, m->m_data, m->m_len,
1037 pmap_kernel(), flags, &lastaddr, &nsegs);
1038 map->len += m->m_len;
1039 }
1040 }
1041 } else {
1042 error = EINVAL;
1043 }
1044
1045 if (error) {
1046 /*
1047 * force "no valid mappings" on error in callback.
1048 */
1049 (*callback)(callback_arg, NULL, 0, 0, error);
1050 } else {
1051 (*callback)(callback_arg, dmat->segments, nsegs + 1,
1052 m0->m_pkthdr.len, error);
1053 }
1054 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1055 __func__, dmat, dmat->flags, error, nsegs + 1);
1056
1057 return (error);
1058}
1059
1060int
1061bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
1062 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
1063 int flags)
1064{
1065 int error = 0;
1066 M_ASSERTPKTHDR(m0);
1067
1068 flags |= BUS_DMA_NOWAIT;
1069 *nsegs = -1;
1070 map->flags &= ~DMAMAP_TYPE_MASK;
1071 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
1072 map->buffer = m0;
1073 map->len = 0;
1074 if (m0->m_pkthdr.len <= dmat->maxsize) {
1075 vm_offset_t lastaddr = 0;
1076 struct mbuf *m;
1077
1078 for (m = m0; m != NULL && error == 0; m = m->m_next) {
1079 if (m->m_len > 0) {
1080 error = bus_dmamap_load_buffer(dmat, segs, map,
1081 m->m_data, m->m_len,
1082 pmap_kernel(), flags, &lastaddr,
1083 nsegs);
1084 map->len += m->m_len;
1085 }
1086 }
1087 } else {
1088 error = EINVAL;
1089 }
1090
1091 /* XXX FIXME: Having to increment nsegs is really annoying */
1092 ++*nsegs;
1093 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1094 __func__, dmat, dmat->flags, error, *nsegs);
1095 return (error);
1096}
1097
1098/*
1099 * Like bus_dmamap_load(), but for uios.
1100 */
1101int
1102bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
1103 bus_dmamap_callback2_t *callback, void *callback_arg,
1104 int flags)
1105{
1106 vm_offset_t lastaddr = 0;
1107 int nsegs, i, error;
1108 bus_size_t resid;
1109 struct iovec *iov;
1110 struct pmap *pmap;
1111
1112 resid = uio->uio_resid;
1113 iov = uio->uio_iov;
1114 map->flags &= ~DMAMAP_TYPE_MASK;
1115 map->flags |= DMAMAP_UIO;
1116 map->buffer = uio;
1117 map->len = 0;
1118
1119 if (uio->uio_segflg == UIO_USERSPACE) {
1120 KASSERT(uio->uio_td != NULL,
1121 ("bus_dmamap_load_uio: USERSPACE but no proc"));
1122 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
1123 } else
1124 pmap = kernel_pmap;
1125
1126 error = 0;
1127 nsegs = -1;
1128 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
1129 /*
1130 * Now at the first iovec to load. Load each iovec
1131 * until we have exhausted the residual count.
1132 */
1133 bus_size_t minlen =
1134 resid < iov[i].iov_len ? resid : iov[i].iov_len;
1135 caddr_t addr = (caddr_t) iov[i].iov_base;
1136
1137 if (minlen > 0) {
1138 error = bus_dmamap_load_buffer(dmat, dmat->segments,
1139 map, addr, minlen, pmap, flags, &lastaddr, &nsegs);
1140
1141 map->len += minlen;
1142 resid -= minlen;
1143 }
1144 }
1145
1146 if (error) {
1147 /*
1148 * force "no valid mappings" on error in callback.
1149 */
1150 (*callback)(callback_arg, NULL, 0, 0, error);
1151 } else {
1152 (*callback)(callback_arg, dmat->segments, nsegs+1,
1153 uio->uio_resid, error);
1154 }
1155
1156 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1157 __func__, dmat, dmat->flags, error, nsegs + 1);
1158 return (error);
1159}
1160
1161/*
1162 * Release the mapping held by map.
1163 */
1164void
1165_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
1166{
1167 struct bounce_page *bpage;
1168
1169 map->flags &= ~DMAMAP_TYPE_MASK;
1170 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1171 STAILQ_REMOVE_HEAD(&map->bpages, links);
1172 free_bounce_page(dmat, bpage);
1173 }
1174 return;
1175}
1176
1177static void
1178bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned)
1179{
1180 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
1181 register_t s;
1182 int partial;
1183
1184 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) {
1185 cpu_dcache_wb_range((vm_offset_t)buf, len);
1186 cpu_l2cache_wb_range((vm_offset_t)buf, len);
1187 }
1188
1189 /*
1190 * If the caller promises the buffer is properly aligned to a cache line
1191 * (even if the call parms make it look like it isn't) we can avoid
1192 * attempting to preserve the non-DMA part of the cache line in the
1193 * POSTREAD case, but we MUST still do a writeback in the PREREAD case.
1194 *
1195 * This covers the case of mbufs, where we know how they're aligned and
1196 * know the CPU doesn't touch the header in front of the DMA data area
1197 * during the IO, but it may have touched it right before invoking the
1198 * sync, so a PREREAD writeback is required.
1199 *
1200 * It also handles buffers we created in bus_dmamem_alloc(), which are
1201 * always aligned and padded to cache line size even if the IO length
1202 * isn't a multiple of cache line size. In this case the PREREAD
1203 * writeback probably isn't required, but it's harmless.
1204 */
1205 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask;
1206
1207 if (op & BUS_DMASYNC_PREREAD) {
1208 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) {
1209 cpu_dcache_inv_range((vm_offset_t)buf, len);
1210 cpu_l2cache_inv_range((vm_offset_t)buf, len);
1211 } else {
1212 cpu_dcache_wbinv_range((vm_offset_t)buf, len);
1213 cpu_l2cache_wbinv_range((vm_offset_t)buf, len);
1214 }
1215 }
1216 if (op & BUS_DMASYNC_POSTREAD) {
1217 if (partial && !bufaligned) {
1218 s = intr_disable();
1219 if ((vm_offset_t)buf & arm_dcache_align_mask)
1220 memcpy(_tmp_cl, (void *)((vm_offset_t)buf &
1221 ~arm_dcache_align_mask),
1222 (vm_offset_t)buf & arm_dcache_align_mask);
1223 if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
1224 memcpy(_tmp_clend,
1225 (void *)((vm_offset_t)buf + len),
1226 arm_dcache_align - (((vm_offset_t)(buf) +
1227 len) & arm_dcache_align_mask));
1228 }
1229 cpu_dcache_inv_range((vm_offset_t)buf, len);
1230 cpu_l2cache_inv_range((vm_offset_t)buf, len);
1231 if (partial && !bufaligned) {
1232 if ((vm_offset_t)buf & arm_dcache_align_mask)
1233 memcpy((void *)((vm_offset_t)buf &
1234 ~arm_dcache_align_mask), _tmp_cl,
1235 (vm_offset_t)buf & arm_dcache_align_mask);
1236 if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
1237 memcpy((void *)((vm_offset_t)buf + len),
1238 _tmp_clend, arm_dcache_align -
1239 (((vm_offset_t)(buf) + len) &
1240 arm_dcache_align_mask));
1241 intr_restore(s);
1242 }
1243 }
1244}
1245
1246static void
1247_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1248{
1249 struct bounce_page *bpage;
1250
1251 STAILQ_FOREACH(bpage, &map->bpages, links) {
1252 if (op & BUS_DMASYNC_PREWRITE) {
1253 bcopy((void *)bpage->datavaddr,
1254 (void *)(bpage->vaddr_nocache != 0 ?
1255 bpage->vaddr_nocache : bpage->vaddr),
1256 bpage->datacount);
1257 if (bpage->vaddr_nocache == 0) {
1258 cpu_dcache_wb_range(bpage->vaddr,
1259 bpage->datacount);
1260 cpu_l2cache_wb_range(bpage->vaddr,
1261 bpage->datacount);
1262 }
1263 dmat->bounce_zone->total_bounced++;
1264 }
1265 if (op & BUS_DMASYNC_POSTREAD) {
1266 if (bpage->vaddr_nocache == 0) {
1267 cpu_dcache_inv_range(bpage->vaddr,
1268 bpage->datacount);
1269 cpu_l2cache_inv_range(bpage->vaddr,
1270 bpage->datacount);
1271 }
1272 bcopy((void *)(bpage->vaddr_nocache != 0 ?
1273 bpage->vaddr_nocache : bpage->vaddr),
1274 (void *)bpage->datavaddr, bpage->datacount);
1275 dmat->bounce_zone->total_bounced++;
1276 }
1277 }
1278}
1279
1280static __inline int
1281_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
1282{
1283 struct bounce_page *bpage;
1284
1285 STAILQ_FOREACH(bpage, &map->bpages, links) {
1286 if ((vm_offset_t)buf >= bpage->datavaddr &&
1287 (vm_offset_t)buf + len <= bpage->datavaddr +
1288 bpage->datacount)
1289 return (1);
1290 }
1291 return (0);
1292
1293}
1294
1295void
1296_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1297{
1298 struct mbuf *m;
1299 struct uio *uio;
1300 int resid;
1301 struct iovec *iov;
1302 int bufaligned;
1303
1304 if (op == BUS_DMASYNC_POSTWRITE)
1305 return;
1306 if (map->flags & DMAMAP_COHERENT)
1307 goto drain;
1308 if (STAILQ_FIRST(&map->bpages))
1309 _bus_dmamap_sync_bp(dmat, map, op);
1310 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
1311 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED);
1312 switch(map->flags & DMAMAP_TYPE_MASK) {
1313 case DMAMAP_LINEAR:
1314 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
1315 bus_dmamap_sync_buf(map->buffer, map->len, op,
1316 bufaligned);
1317 break;
1318 case DMAMAP_MBUF:
1319 m = map->buffer;
1320 while (m) {
1321 if (m->m_len > 0 &&
1322 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
1323 bus_dmamap_sync_buf(m->m_data, m->m_len, op,
1324 bufaligned);
1325 m = m->m_next;
1326 }
1327 break;
1328 case DMAMAP_UIO:
1329 uio = map->buffer;
1330 iov = uio->uio_iov;
1331 resid = uio->uio_resid;
1332 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1333 bus_size_t minlen = resid < iov[i].iov_len ? resid :
1334 iov[i].iov_len;
1335 if (minlen > 0) {
1336 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
1337 minlen))
1338 bus_dmamap_sync_buf(iov[i].iov_base,
1339 minlen, op, bufaligned);
1340 resid -= minlen;
1341 }
1342 }
1343 break;
1344 default:
1345 break;
1346 }
1347
1348drain:
1349
1350 cpu_drain_writebuf();
1351}
1352
1353static void
1354init_bounce_pages(void *dummy __unused)
1355{
1356
1357 total_bpages = 0;
1358 STAILQ_INIT(&bounce_zone_list);
1359 STAILQ_INIT(&bounce_map_waitinglist);
1360 STAILQ_INIT(&bounce_map_callbacklist);
1361 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
1362}
1363SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
1364
1365static struct sysctl_ctx_list *
1366busdma_sysctl_tree(struct bounce_zone *bz)
1367{
1368 return (&bz->sysctl_tree);
1369}
1370
1371static struct sysctl_oid *
1372busdma_sysctl_tree_top(struct bounce_zone *bz)
1373{
1374 return (bz->sysctl_tree_top);
1375}
1376
1377static int
1378alloc_bounce_zone(bus_dma_tag_t dmat)
1379{
1380 struct bounce_zone *bz;
1381
1382 /* Check to see if we already have a suitable zone */
1383 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1384 if ((dmat->alignment <= bz->alignment)
1385 && (dmat->lowaddr >= bz->lowaddr)) {
1386 dmat->bounce_zone = bz;
1387 return (0);
1388 }
1389 }
1390
1391 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
1392 M_NOWAIT | M_ZERO)) == NULL)
1393 return (ENOMEM);
1394
1395 STAILQ_INIT(&bz->bounce_page_list);
1396 bz->free_bpages = 0;
1397 bz->reserved_bpages = 0;
1398 bz->active_bpages = 0;
1399 bz->lowaddr = dmat->lowaddr;
1400 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1401 bz->map_count = 0;
1402 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1403 busdma_zonecount++;
1404 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1405 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1406 dmat->bounce_zone = bz;
1407
1408 sysctl_ctx_init(&bz->sysctl_tree);
1409 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1410 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1411 CTLFLAG_RD, 0, "");
1412 if (bz->sysctl_tree_top == NULL) {
1413 sysctl_ctx_free(&bz->sysctl_tree);
1414 return (0); /* XXX error code? */
1415 }
1416
1417 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1418 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1419 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1420 "Total bounce pages");
1421 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1422 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1423 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1424 "Free bounce pages");
1425 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1426 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1427 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1428 "Reserved bounce pages");
1429 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1430 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1431 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1432 "Active bounce pages");
1433 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1434 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1435 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1436 "Total bounce requests");
1437 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1438 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1439 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1440 "Total bounce requests that were deferred");
1441 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1442 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1443 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1444 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1445 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1446 "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
1447
1448 return (0);
1449}
1450
1451static int
1452alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1453{
1454 struct bounce_zone *bz;
1455 int count;
1456
1457 bz = dmat->bounce_zone;
1458 count = 0;
1459 while (numpages > 0) {
1460 struct bounce_page *bpage;
1461
1462 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1463 M_NOWAIT | M_ZERO);
1464
1465 if (bpage == NULL)
1466 break;
1467 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1468 M_NOWAIT, 0ul,
1469 bz->lowaddr,
1470 PAGE_SIZE,
1471 0);
1472 if (bpage->vaddr == 0) {
1473 free(bpage, M_DEVBUF);
1474 break;
1475 }
1476 bpage->busaddr = pmap_kextract(bpage->vaddr);
1477 bpage->vaddr_nocache = (vm_offset_t)arm_remap_nocache(
1478 (void *)bpage->vaddr, PAGE_SIZE);
1479 mtx_lock(&bounce_lock);
1480 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1481 total_bpages++;
1482 bz->total_bpages++;
1483 bz->free_bpages++;
1484 mtx_unlock(&bounce_lock);
1485 count++;
1486 numpages--;
1487 }
1488 return (count);
1489}
1490
1491static int
1492reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1493{
1494 struct bounce_zone *bz;
1495 int pages;
1496
1497 mtx_assert(&bounce_lock, MA_OWNED);
1498 bz = dmat->bounce_zone;
1499 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1500 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1501 return (map->pagesneeded - (map->pagesreserved + pages));
1502 bz->free_bpages -= pages;
1503 bz->reserved_bpages += pages;
1504 map->pagesreserved += pages;
1505 pages = map->pagesneeded - map->pagesreserved;
1506
1507 return (pages);
1508}
1509
1510static bus_addr_t
1511add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1512 bus_size_t size)
1513{
1514 struct bounce_zone *bz;
1515 struct bounce_page *bpage;
1516
1517 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1518 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
1519
1520 bz = dmat->bounce_zone;
1521 if (map->pagesneeded == 0)
1522 panic("add_bounce_page: map doesn't need any pages");
1523 map->pagesneeded--;
1524
1525 if (map->pagesreserved == 0)
1526 panic("add_bounce_page: map doesn't need any pages");
1527 map->pagesreserved--;
1528
1529 mtx_lock(&bounce_lock);
1530 bpage = STAILQ_FIRST(&bz->bounce_page_list);
1531 if (bpage == NULL)
1532 panic("add_bounce_page: free page list is empty");
1533
1534 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1535 bz->reserved_bpages--;
1536 bz->active_bpages++;
1537 mtx_unlock(&bounce_lock);
1538
1539 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1540 /* Page offset needs to be preserved. */
1541 bpage->vaddr |= vaddr & PAGE_MASK;
1542 bpage->busaddr |= vaddr & PAGE_MASK;
1543 }
1544 bpage->datavaddr = vaddr;
1545 bpage->datacount = size;
1546 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1547 return (bpage->busaddr);
1548}
1549
1550static void
1551free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1552{
1553 struct bus_dmamap *map;
1554 struct bounce_zone *bz;
1555
1556 bz = dmat->bounce_zone;
1557 bpage->datavaddr = 0;
1558 bpage->datacount = 0;
1559 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1560 /*
1561 * Reset the bounce page to start at offset 0. Other uses
1562 * of this bounce page may need to store a full page of
1563 * data and/or assume it starts on a page boundary.
1564 */
1565 bpage->vaddr &= ~PAGE_MASK;
1566 bpage->busaddr &= ~PAGE_MASK;
1567 }
1568
1569 mtx_lock(&bounce_lock);
1570 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1571 bz->free_bpages++;
1572 bz->active_bpages--;
1573 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1574 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1575 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1576 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1577 map, links);
1578 busdma_swi_pending = 1;
1579 bz->total_deferred++;
1580 swi_sched(vm_ih, 0);
1581 }
1582 }
1583 mtx_unlock(&bounce_lock);
1584}
1585
1586void
1587busdma_swi(void)
1588{
1589 bus_dma_tag_t dmat;
1590 struct bus_dmamap *map;
1591
1592 mtx_lock(&bounce_lock);
1593 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1594 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1595 mtx_unlock(&bounce_lock);
1596 dmat = map->dmat;
1597 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1598 bus_dmamap_load(map->dmat, map, map->buffer, map->len,
1599 map->callback, map->callback_arg, /*flags*/0);
1600 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1601 mtx_lock(&bounce_lock);
1602 }
1603 mtx_unlock(&bounce_lock);
1604}
| 588
589 /*
590 * Bouncing might be required if the driver asks for an active
591 * exclusion region, a data alignment that is stricter than 1, and/or
592 * an active address boundary.
593 */
594 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
595
596 /* Must bounce */
597 struct bounce_zone *bz;
598 int maxpages;
599
600 if (dmat->bounce_zone == NULL) {
601 if ((error = alloc_bounce_zone(dmat)) != 0) {
602 uma_zfree(dmamap_zone, map);
603 *mapp = NULL;
604 return (error);
605 }
606 }
607 bz = dmat->bounce_zone;
608
609 /* Initialize the new map */
610 STAILQ_INIT(&((*mapp)->bpages));
611
612 /*
613 * Attempt to add pages to our pool on a per-instance
614 * basis up to a sane limit.
615 */
616 maxpages = MAX_BPAGES;
617 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
618 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
619 int pages;
620
621 pages = MAX(atop(dmat->maxsize), 1);
622 pages = MIN(maxpages - bz->total_bpages, pages);
623 pages = MAX(pages, 1);
624 if (alloc_bounce_pages(dmat, pages) < pages)
625 error = ENOMEM;
626
627 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
628 if (error == 0)
629 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
630 } else {
631 error = 0;
632 }
633 }
634 bz->map_count++;
635 }
636 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
637 __func__, dmat, dmat->flags, error);
638
639 return (0);
640}
641
642/*
643 * Destroy a handle for mapping from kva/uva/physical
644 * address space into bus device space.
645 */
646int
647bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
648{
649
650 if (STAILQ_FIRST(&map->bpages) != NULL) {
651 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
652 __func__, dmat, EBUSY);
653 return (EBUSY);
654 }
655 uma_zfree(dmamap_zone, map);
656 if (dmat->bounce_zone)
657 dmat->bounce_zone->map_count--;
658 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
659 return (0);
660}
661
662/*
663 * Allocate a piece of memory that can be efficiently mapped into bus device
664 * space based on the constraints listed in the dma tag. Returns a pointer to
665 * the allocated memory, and a pointer to an associated bus_dmamap.
666 */
667int
668bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags,
669 bus_dmamap_t *mapp)
670{
671 void * vaddr;
672 struct busdma_bufzone *bufzone;
673 busdma_bufalloc_t ba;
674 bus_dmamap_t map;
675 int mflags;
676 vm_memattr_t memattr;
677
678 if (flags & BUS_DMA_NOWAIT)
679 mflags = M_NOWAIT;
680 else
681 mflags = M_WAITOK;
682
683 /*
684 * If the tag's segments haven't been allocated yet we need to do it
685 * now, because we can't sleep for resources at map load time.
686 */
687 if (dmat->segments == NULL)
688 dmat->segments = malloc(dmat->nsegments *
689 sizeof(*dmat->segments), M_DEVBUF, mflags);
690
691 map = uma_zalloc_arg(dmamap_zone, dmat, mflags);
692 if (map == NULL)
693 return (ENOMEM);
694
695 if (flags & BUS_DMA_COHERENT) {
696 memattr = VM_MEMATTR_UNCACHEABLE;
697 ba = coherent_allocator;
698 map->flags |= DMAMAP_COHERENT;
699 } else {
700 memattr = VM_MEMATTR_DEFAULT;
701 ba = standard_allocator;
702 }
703 /* All buffers we allocate are cache-aligned. */
704 map->flags |= DMAMAP_CACHE_ALIGNED;
705
706 if (flags & BUS_DMA_ZERO)
707 mflags |= M_ZERO;
708
709 /*
710 * Try to find a bufzone in the allocator that holds a cache of buffers
711 * of the right size for this request. If the buffer is too big to be
712 * held in the allocator cache, this returns NULL.
713 */
714 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
715
716 /*
717 * Allocate the buffer from the uma(9) allocator if...
718 * - It's small enough to be in the allocator (bufzone not NULL).
719 * - The alignment constraint isn't larger than the allocation size
720 * (the allocator aligns buffers to their size boundaries).
721 * - There's no need to handle lowaddr/highaddr exclusion zones.
722 * else allocate non-contiguous pages if...
723 * - The page count that could get allocated doesn't exceed nsegments.
724 * - The alignment constraint isn't larger than a page boundary.
725 * - There are no boundary-crossing constraints.
726 * else allocate a block of contiguous pages because one or more of the
727 * constraints is something that only the contig allocator can fulfill.
728 */
729 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
730 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
731 vaddr = uma_zalloc(bufzone->umazone, mflags);
732 } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
733 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) {
734 vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize,
735 mflags, 0, dmat->lowaddr, memattr);
736 } else {
737 vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize,
738 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
739 memattr);
740 }
741
742 if (vaddr == NULL) {
743 uma_zfree(dmamap_zone, map);
744 map = NULL;
745 }
746
747 *vaddrp = vaddr;
748 *mapp = map;
749
750 return (vaddr == NULL ? ENOMEM : 0);
751}
752
753/*
754 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with
755 * its associated map.
756 */
757void
758bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
759{
760 struct busdma_bufzone *bufzone;
761 busdma_bufalloc_t ba;
762
763 if (map->flags & DMAMAP_COHERENT)
764 ba = coherent_allocator;
765 else
766 ba = standard_allocator;
767 uma_zfree(dmamap_zone, map);
768
769 /* Be careful not to access map from here on. */
770
771 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
772
773 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
774 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
775 uma_zfree(bufzone->umazone, vaddr);
776 else
777 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize);
778}
779
780static int
781_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
782 void *buf, bus_size_t buflen, int flags)
783{
784 vm_offset_t vaddr;
785 vm_offset_t vendaddr;
786 bus_addr_t paddr;
787
788 if ((map->pagesneeded == 0)) {
789 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
790 dmat->lowaddr, dmat->boundary, dmat->alignment);
791 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
792 map, map->pagesneeded);
793 /*
794 * Count the number of bounce pages
795 * needed in order to complete this transfer
796 */
797 vaddr = trunc_page((vm_offset_t)buf);
798 vendaddr = (vm_offset_t)buf + buflen;
799
800 while (vaddr < vendaddr) {
801 if (__predict_true(pmap == pmap_kernel()))
802 paddr = pmap_kextract(vaddr);
803 else
804 paddr = pmap_extract(pmap, vaddr);
805 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
806 run_filter(dmat, paddr) != 0)
807 map->pagesneeded++;
808 vaddr += PAGE_SIZE;
809 }
810 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
811 }
812
813 /* Reserve Necessary Bounce Pages */
814 if (map->pagesneeded != 0) {
815 mtx_lock(&bounce_lock);
816 if (flags & BUS_DMA_NOWAIT) {
817 if (reserve_bounce_pages(dmat, map, 0) != 0) {
818 mtx_unlock(&bounce_lock);
819 return (ENOMEM);
820 }
821 } else {
822 if (reserve_bounce_pages(dmat, map, 1) != 0) {
823 /* Queue us for resources */
824 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
825 map, links);
826 mtx_unlock(&bounce_lock);
827 return (EINPROGRESS);
828 }
829 }
830 mtx_unlock(&bounce_lock);
831 }
832
833 return (0);
834}
835
836/*
837 * Utility function to load a linear buffer. lastaddrp holds state
838 * between invocations (for multiple-buffer loads). segp contains
839 * the starting segment on entrance, and the ending segment on exit.
840 * first indicates if this is the first invocation of this function.
841 */
842static __inline int
843bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs,
844 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap,
845 int flags, vm_offset_t *lastaddrp, int *segp)
846{
847 bus_size_t sgsize;
848 bus_addr_t curaddr, lastaddr, baddr, bmask;
849 vm_offset_t vaddr = (vm_offset_t)buf;
850 int seg;
851 int error = 0;
852 pd_entry_t *pde;
853 pt_entry_t pte;
854 pt_entry_t *ptep;
855
856 lastaddr = *lastaddrp;
857 bmask = ~(dmat->boundary - 1);
858
859 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
860 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen,
861 flags);
862 if (error)
863 return (error);
864 }
865 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
866 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
867
868 for (seg = *segp; buflen > 0 ; ) {
869 /*
870 * Get the physical address for this segment.
871 *
872 * XXX Don't support checking for coherent mappings
873 * XXX in user address space.
874 */
875 if (__predict_true(pmap == pmap_kernel())) {
876 if (pmap_get_pde_pte(pmap, vaddr, &pde, &ptep) == FALSE)
877 return (EFAULT);
878
879 if (__predict_false(pmap_pde_section(pde))) {
880 if (*pde & L1_S_SUPERSEC)
881 curaddr = (*pde & L1_SUP_FRAME) |
882 (vaddr & L1_SUP_OFFSET);
883 else
884 curaddr = (*pde & L1_S_FRAME) |
885 (vaddr & L1_S_OFFSET);
886 } else {
887 pte = *ptep;
888 KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV,
889 ("INV type"));
890 if (__predict_false((pte & L2_TYPE_MASK)
891 == L2_TYPE_L)) {
892 curaddr = (pte & L2_L_FRAME) |
893 (vaddr & L2_L_OFFSET);
894 } else {
895 curaddr = (pte & L2_S_FRAME) |
896 (vaddr & L2_S_OFFSET);
897 }
898 }
899 } else {
900 curaddr = pmap_extract(pmap, vaddr);
901 map->flags &= ~DMAMAP_COHERENT;
902 }
903
904 /*
905 * Compute the segment size, and adjust counts.
906 */
907 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
908 if (sgsize > dmat->maxsegsz)
909 sgsize = dmat->maxsegsz;
910 if (buflen < sgsize)
911 sgsize = buflen;
912
913 /*
914 * Make sure we don't cross any boundaries.
915 */
916 if (dmat->boundary > 0) {
917 baddr = (curaddr + dmat->boundary) & bmask;
918 if (sgsize > (baddr - curaddr))
919 sgsize = (baddr - curaddr);
920 }
921 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
922 map->pagesneeded != 0 && run_filter(dmat, curaddr))
923 curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
924
925 if (dmat->ranges) {
926 struct arm32_dma_range *dr;
927
928 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
929 curaddr);
930 if (dr == NULL)
931 return (EINVAL);
932 /*
933 * In a valid DMA range. Translate the physical
934 * memory address to an address in the DMA window.
935 */
936 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
937
938 }
939
940 /*
941 * Insert chunk into a segment, coalescing with
942 * the previous segment if possible.
943 */
944 if (seg >= 0 && curaddr == lastaddr &&
945 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
946 (dmat->boundary == 0 ||
947 (segs[seg].ds_addr & bmask) ==
948 (curaddr & bmask))) {
949 segs[seg].ds_len += sgsize;
950 goto segdone;
951 } else {
952 if (++seg >= dmat->nsegments)
953 break;
954 segs[seg].ds_addr = curaddr;
955 segs[seg].ds_len = sgsize;
956 }
957 if (error)
958 break;
959segdone:
960 lastaddr = curaddr + sgsize;
961 vaddr += sgsize;
962 buflen -= sgsize;
963 }
964
965 *segp = seg;
966 *lastaddrp = lastaddr;
967
968 /*
969 * Did we fit?
970 */
971 if (buflen != 0)
972 error = EFBIG; /* XXX better return value here? */
973 return (error);
974}
975
976/*
977 * Map the buffer buf into bus space using the dmamap map.
978 */
979int
980bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
981 bus_size_t buflen, bus_dmamap_callback_t *callback,
982 void *callback_arg, int flags)
983{
984 vm_offset_t lastaddr = 0;
985 int error, nsegs = -1;
986
987 KASSERT(dmat != NULL, ("dmatag is NULL"));
988 KASSERT(map != NULL, ("dmamap is NULL"));
989 map->callback = callback;
990 map->callback_arg = callback_arg;
991 map->flags &= ~DMAMAP_TYPE_MASK;
992 map->flags |= DMAMAP_LINEAR;
993 map->buffer = buf;
994 map->len = buflen;
995 error = bus_dmamap_load_buffer(dmat,
996 dmat->segments, map, buf, buflen, kernel_pmap,
997 flags, &lastaddr, &nsegs);
998 if (error == EINPROGRESS)
999 return (error);
1000 if (error)
1001 (*callback)(callback_arg, NULL, 0, error);
1002 else
1003 (*callback)(callback_arg, dmat->segments, nsegs + 1, error);
1004
1005 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1006 __func__, dmat, dmat->flags, nsegs + 1, error);
1007
1008 return (error);
1009}
1010
1011/*
1012 * Like bus_dmamap_load(), but for mbufs.
1013 *
1014 * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs.
1015 *
1016 * We know that the way the system allocates and uses mbufs implies that we can
1017 * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line
1018 * flushes. Even though the flush may reference the data area within the mbuf
1019 * that isn't aligned to a cache line, we know the overall mbuf itself is
1020 * properly aligned, and we know that the CPU will not touch the header fields
1021 * before the data area while the DMA is in progress.
1022 */
1023int
1024bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
1025 bus_dmamap_callback2_t *callback, void *callback_arg,
1026 int flags)
1027{
1028 int nsegs = -1, error = 0;
1029
1030 M_ASSERTPKTHDR(m0);
1031
1032 flags |= BUS_DMA_NOWAIT;
1033 map->flags &= ~DMAMAP_TYPE_MASK;
1034 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
1035 map->buffer = m0;
1036 map->len = 0;
1037 if (m0->m_pkthdr.len <= dmat->maxsize) {
1038 vm_offset_t lastaddr = 0;
1039 struct mbuf *m;
1040
1041 for (m = m0; m != NULL && error == 0; m = m->m_next) {
1042 if (m->m_len > 0) {
1043 error = bus_dmamap_load_buffer(dmat,
1044 dmat->segments, map, m->m_data, m->m_len,
1045 pmap_kernel(), flags, &lastaddr, &nsegs);
1046 map->len += m->m_len;
1047 }
1048 }
1049 } else {
1050 error = EINVAL;
1051 }
1052
1053 if (error) {
1054 /*
1055 * force "no valid mappings" on error in callback.
1056 */
1057 (*callback)(callback_arg, NULL, 0, 0, error);
1058 } else {
1059 (*callback)(callback_arg, dmat->segments, nsegs + 1,
1060 m0->m_pkthdr.len, error);
1061 }
1062 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1063 __func__, dmat, dmat->flags, error, nsegs + 1);
1064
1065 return (error);
1066}
1067
1068int
1069bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
1070 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
1071 int flags)
1072{
1073 int error = 0;
1074 M_ASSERTPKTHDR(m0);
1075
1076 flags |= BUS_DMA_NOWAIT;
1077 *nsegs = -1;
1078 map->flags &= ~DMAMAP_TYPE_MASK;
1079 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
1080 map->buffer = m0;
1081 map->len = 0;
1082 if (m0->m_pkthdr.len <= dmat->maxsize) {
1083 vm_offset_t lastaddr = 0;
1084 struct mbuf *m;
1085
1086 for (m = m0; m != NULL && error == 0; m = m->m_next) {
1087 if (m->m_len > 0) {
1088 error = bus_dmamap_load_buffer(dmat, segs, map,
1089 m->m_data, m->m_len,
1090 pmap_kernel(), flags, &lastaddr,
1091 nsegs);
1092 map->len += m->m_len;
1093 }
1094 }
1095 } else {
1096 error = EINVAL;
1097 }
1098
1099 /* XXX FIXME: Having to increment nsegs is really annoying */
1100 ++*nsegs;
1101 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1102 __func__, dmat, dmat->flags, error, *nsegs);
1103 return (error);
1104}
1105
1106/*
1107 * Like bus_dmamap_load(), but for uios.
1108 */
1109int
1110bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
1111 bus_dmamap_callback2_t *callback, void *callback_arg,
1112 int flags)
1113{
1114 vm_offset_t lastaddr = 0;
1115 int nsegs, i, error;
1116 bus_size_t resid;
1117 struct iovec *iov;
1118 struct pmap *pmap;
1119
1120 resid = uio->uio_resid;
1121 iov = uio->uio_iov;
1122 map->flags &= ~DMAMAP_TYPE_MASK;
1123 map->flags |= DMAMAP_UIO;
1124 map->buffer = uio;
1125 map->len = 0;
1126
1127 if (uio->uio_segflg == UIO_USERSPACE) {
1128 KASSERT(uio->uio_td != NULL,
1129 ("bus_dmamap_load_uio: USERSPACE but no proc"));
1130 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
1131 } else
1132 pmap = kernel_pmap;
1133
1134 error = 0;
1135 nsegs = -1;
1136 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
1137 /*
1138 * Now at the first iovec to load. Load each iovec
1139 * until we have exhausted the residual count.
1140 */
1141 bus_size_t minlen =
1142 resid < iov[i].iov_len ? resid : iov[i].iov_len;
1143 caddr_t addr = (caddr_t) iov[i].iov_base;
1144
1145 if (minlen > 0) {
1146 error = bus_dmamap_load_buffer(dmat, dmat->segments,
1147 map, addr, minlen, pmap, flags, &lastaddr, &nsegs);
1148
1149 map->len += minlen;
1150 resid -= minlen;
1151 }
1152 }
1153
1154 if (error) {
1155 /*
1156 * force "no valid mappings" on error in callback.
1157 */
1158 (*callback)(callback_arg, NULL, 0, 0, error);
1159 } else {
1160 (*callback)(callback_arg, dmat->segments, nsegs+1,
1161 uio->uio_resid, error);
1162 }
1163
1164 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
1165 __func__, dmat, dmat->flags, error, nsegs + 1);
1166 return (error);
1167}
1168
1169/*
1170 * Release the mapping held by map.
1171 */
1172void
1173_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
1174{
1175 struct bounce_page *bpage;
1176
1177 map->flags &= ~DMAMAP_TYPE_MASK;
1178 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1179 STAILQ_REMOVE_HEAD(&map->bpages, links);
1180 free_bounce_page(dmat, bpage);
1181 }
1182 return;
1183}
1184
1185static void
1186bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned)
1187{
1188 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
1189 register_t s;
1190 int partial;
1191
1192 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) {
1193 cpu_dcache_wb_range((vm_offset_t)buf, len);
1194 cpu_l2cache_wb_range((vm_offset_t)buf, len);
1195 }
1196
1197 /*
1198 * If the caller promises the buffer is properly aligned to a cache line
1199 * (even if the call parms make it look like it isn't) we can avoid
1200 * attempting to preserve the non-DMA part of the cache line in the
1201 * POSTREAD case, but we MUST still do a writeback in the PREREAD case.
1202 *
1203 * This covers the case of mbufs, where we know how they're aligned and
1204 * know the CPU doesn't touch the header in front of the DMA data area
1205 * during the IO, but it may have touched it right before invoking the
1206 * sync, so a PREREAD writeback is required.
1207 *
1208 * It also handles buffers we created in bus_dmamem_alloc(), which are
1209 * always aligned and padded to cache line size even if the IO length
1210 * isn't a multiple of cache line size. In this case the PREREAD
1211 * writeback probably isn't required, but it's harmless.
1212 */
1213 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask;
1214
1215 if (op & BUS_DMASYNC_PREREAD) {
1216 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) {
1217 cpu_dcache_inv_range((vm_offset_t)buf, len);
1218 cpu_l2cache_inv_range((vm_offset_t)buf, len);
1219 } else {
1220 cpu_dcache_wbinv_range((vm_offset_t)buf, len);
1221 cpu_l2cache_wbinv_range((vm_offset_t)buf, len);
1222 }
1223 }
1224 if (op & BUS_DMASYNC_POSTREAD) {
1225 if (partial && !bufaligned) {
1226 s = intr_disable();
1227 if ((vm_offset_t)buf & arm_dcache_align_mask)
1228 memcpy(_tmp_cl, (void *)((vm_offset_t)buf &
1229 ~arm_dcache_align_mask),
1230 (vm_offset_t)buf & arm_dcache_align_mask);
1231 if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
1232 memcpy(_tmp_clend,
1233 (void *)((vm_offset_t)buf + len),
1234 arm_dcache_align - (((vm_offset_t)(buf) +
1235 len) & arm_dcache_align_mask));
1236 }
1237 cpu_dcache_inv_range((vm_offset_t)buf, len);
1238 cpu_l2cache_inv_range((vm_offset_t)buf, len);
1239 if (partial && !bufaligned) {
1240 if ((vm_offset_t)buf & arm_dcache_align_mask)
1241 memcpy((void *)((vm_offset_t)buf &
1242 ~arm_dcache_align_mask), _tmp_cl,
1243 (vm_offset_t)buf & arm_dcache_align_mask);
1244 if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
1245 memcpy((void *)((vm_offset_t)buf + len),
1246 _tmp_clend, arm_dcache_align -
1247 (((vm_offset_t)(buf) + len) &
1248 arm_dcache_align_mask));
1249 intr_restore(s);
1250 }
1251 }
1252}
1253
1254static void
1255_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1256{
1257 struct bounce_page *bpage;
1258
1259 STAILQ_FOREACH(bpage, &map->bpages, links) {
1260 if (op & BUS_DMASYNC_PREWRITE) {
1261 bcopy((void *)bpage->datavaddr,
1262 (void *)(bpage->vaddr_nocache != 0 ?
1263 bpage->vaddr_nocache : bpage->vaddr),
1264 bpage->datacount);
1265 if (bpage->vaddr_nocache == 0) {
1266 cpu_dcache_wb_range(bpage->vaddr,
1267 bpage->datacount);
1268 cpu_l2cache_wb_range(bpage->vaddr,
1269 bpage->datacount);
1270 }
1271 dmat->bounce_zone->total_bounced++;
1272 }
1273 if (op & BUS_DMASYNC_POSTREAD) {
1274 if (bpage->vaddr_nocache == 0) {
1275 cpu_dcache_inv_range(bpage->vaddr,
1276 bpage->datacount);
1277 cpu_l2cache_inv_range(bpage->vaddr,
1278 bpage->datacount);
1279 }
1280 bcopy((void *)(bpage->vaddr_nocache != 0 ?
1281 bpage->vaddr_nocache : bpage->vaddr),
1282 (void *)bpage->datavaddr, bpage->datacount);
1283 dmat->bounce_zone->total_bounced++;
1284 }
1285 }
1286}
1287
1288static __inline int
1289_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
1290{
1291 struct bounce_page *bpage;
1292
1293 STAILQ_FOREACH(bpage, &map->bpages, links) {
1294 if ((vm_offset_t)buf >= bpage->datavaddr &&
1295 (vm_offset_t)buf + len <= bpage->datavaddr +
1296 bpage->datacount)
1297 return (1);
1298 }
1299 return (0);
1300
1301}
1302
1303void
1304_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1305{
1306 struct mbuf *m;
1307 struct uio *uio;
1308 int resid;
1309 struct iovec *iov;
1310 int bufaligned;
1311
1312 if (op == BUS_DMASYNC_POSTWRITE)
1313 return;
1314 if (map->flags & DMAMAP_COHERENT)
1315 goto drain;
1316 if (STAILQ_FIRST(&map->bpages))
1317 _bus_dmamap_sync_bp(dmat, map, op);
1318 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
1319 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED);
1320 switch(map->flags & DMAMAP_TYPE_MASK) {
1321 case DMAMAP_LINEAR:
1322 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
1323 bus_dmamap_sync_buf(map->buffer, map->len, op,
1324 bufaligned);
1325 break;
1326 case DMAMAP_MBUF:
1327 m = map->buffer;
1328 while (m) {
1329 if (m->m_len > 0 &&
1330 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
1331 bus_dmamap_sync_buf(m->m_data, m->m_len, op,
1332 bufaligned);
1333 m = m->m_next;
1334 }
1335 break;
1336 case DMAMAP_UIO:
1337 uio = map->buffer;
1338 iov = uio->uio_iov;
1339 resid = uio->uio_resid;
1340 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1341 bus_size_t minlen = resid < iov[i].iov_len ? resid :
1342 iov[i].iov_len;
1343 if (minlen > 0) {
1344 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
1345 minlen))
1346 bus_dmamap_sync_buf(iov[i].iov_base,
1347 minlen, op, bufaligned);
1348 resid -= minlen;
1349 }
1350 }
1351 break;
1352 default:
1353 break;
1354 }
1355
1356drain:
1357
1358 cpu_drain_writebuf();
1359}
1360
1361static void
1362init_bounce_pages(void *dummy __unused)
1363{
1364
1365 total_bpages = 0;
1366 STAILQ_INIT(&bounce_zone_list);
1367 STAILQ_INIT(&bounce_map_waitinglist);
1368 STAILQ_INIT(&bounce_map_callbacklist);
1369 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
1370}
1371SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
1372
1373static struct sysctl_ctx_list *
1374busdma_sysctl_tree(struct bounce_zone *bz)
1375{
1376 return (&bz->sysctl_tree);
1377}
1378
1379static struct sysctl_oid *
1380busdma_sysctl_tree_top(struct bounce_zone *bz)
1381{
1382 return (bz->sysctl_tree_top);
1383}
1384
1385static int
1386alloc_bounce_zone(bus_dma_tag_t dmat)
1387{
1388 struct bounce_zone *bz;
1389
1390 /* Check to see if we already have a suitable zone */
1391 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1392 if ((dmat->alignment <= bz->alignment)
1393 && (dmat->lowaddr >= bz->lowaddr)) {
1394 dmat->bounce_zone = bz;
1395 return (0);
1396 }
1397 }
1398
1399 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
1400 M_NOWAIT | M_ZERO)) == NULL)
1401 return (ENOMEM);
1402
1403 STAILQ_INIT(&bz->bounce_page_list);
1404 bz->free_bpages = 0;
1405 bz->reserved_bpages = 0;
1406 bz->active_bpages = 0;
1407 bz->lowaddr = dmat->lowaddr;
1408 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1409 bz->map_count = 0;
1410 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1411 busdma_zonecount++;
1412 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1413 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1414 dmat->bounce_zone = bz;
1415
1416 sysctl_ctx_init(&bz->sysctl_tree);
1417 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1418 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1419 CTLFLAG_RD, 0, "");
1420 if (bz->sysctl_tree_top == NULL) {
1421 sysctl_ctx_free(&bz->sysctl_tree);
1422 return (0); /* XXX error code? */
1423 }
1424
1425 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1426 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1427 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1428 "Total bounce pages");
1429 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1430 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1431 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1432 "Free bounce pages");
1433 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1434 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1435 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1436 "Reserved bounce pages");
1437 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1438 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1439 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1440 "Active bounce pages");
1441 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1442 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1443 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1444 "Total bounce requests");
1445 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1446 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1447 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1448 "Total bounce requests that were deferred");
1449 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1450 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1451 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1452 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1453 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1454 "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
1455
1456 return (0);
1457}
1458
1459static int
1460alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1461{
1462 struct bounce_zone *bz;
1463 int count;
1464
1465 bz = dmat->bounce_zone;
1466 count = 0;
1467 while (numpages > 0) {
1468 struct bounce_page *bpage;
1469
1470 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1471 M_NOWAIT | M_ZERO);
1472
1473 if (bpage == NULL)
1474 break;
1475 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1476 M_NOWAIT, 0ul,
1477 bz->lowaddr,
1478 PAGE_SIZE,
1479 0);
1480 if (bpage->vaddr == 0) {
1481 free(bpage, M_DEVBUF);
1482 break;
1483 }
1484 bpage->busaddr = pmap_kextract(bpage->vaddr);
1485 bpage->vaddr_nocache = (vm_offset_t)arm_remap_nocache(
1486 (void *)bpage->vaddr, PAGE_SIZE);
1487 mtx_lock(&bounce_lock);
1488 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1489 total_bpages++;
1490 bz->total_bpages++;
1491 bz->free_bpages++;
1492 mtx_unlock(&bounce_lock);
1493 count++;
1494 numpages--;
1495 }
1496 return (count);
1497}
1498
1499static int
1500reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1501{
1502 struct bounce_zone *bz;
1503 int pages;
1504
1505 mtx_assert(&bounce_lock, MA_OWNED);
1506 bz = dmat->bounce_zone;
1507 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1508 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1509 return (map->pagesneeded - (map->pagesreserved + pages));
1510 bz->free_bpages -= pages;
1511 bz->reserved_bpages += pages;
1512 map->pagesreserved += pages;
1513 pages = map->pagesneeded - map->pagesreserved;
1514
1515 return (pages);
1516}
1517
1518static bus_addr_t
1519add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1520 bus_size_t size)
1521{
1522 struct bounce_zone *bz;
1523 struct bounce_page *bpage;
1524
1525 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1526 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
1527
1528 bz = dmat->bounce_zone;
1529 if (map->pagesneeded == 0)
1530 panic("add_bounce_page: map doesn't need any pages");
1531 map->pagesneeded--;
1532
1533 if (map->pagesreserved == 0)
1534 panic("add_bounce_page: map doesn't need any pages");
1535 map->pagesreserved--;
1536
1537 mtx_lock(&bounce_lock);
1538 bpage = STAILQ_FIRST(&bz->bounce_page_list);
1539 if (bpage == NULL)
1540 panic("add_bounce_page: free page list is empty");
1541
1542 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1543 bz->reserved_bpages--;
1544 bz->active_bpages++;
1545 mtx_unlock(&bounce_lock);
1546
1547 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1548 /* Page offset needs to be preserved. */
1549 bpage->vaddr |= vaddr & PAGE_MASK;
1550 bpage->busaddr |= vaddr & PAGE_MASK;
1551 }
1552 bpage->datavaddr = vaddr;
1553 bpage->datacount = size;
1554 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1555 return (bpage->busaddr);
1556}
1557
1558static void
1559free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1560{
1561 struct bus_dmamap *map;
1562 struct bounce_zone *bz;
1563
1564 bz = dmat->bounce_zone;
1565 bpage->datavaddr = 0;
1566 bpage->datacount = 0;
1567 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1568 /*
1569 * Reset the bounce page to start at offset 0. Other uses
1570 * of this bounce page may need to store a full page of
1571 * data and/or assume it starts on a page boundary.
1572 */
1573 bpage->vaddr &= ~PAGE_MASK;
1574 bpage->busaddr &= ~PAGE_MASK;
1575 }
1576
1577 mtx_lock(&bounce_lock);
1578 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1579 bz->free_bpages++;
1580 bz->active_bpages--;
1581 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1582 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1583 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1584 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1585 map, links);
1586 busdma_swi_pending = 1;
1587 bz->total_deferred++;
1588 swi_sched(vm_ih, 0);
1589 }
1590 }
1591 mtx_unlock(&bounce_lock);
1592}
1593
1594void
1595busdma_swi(void)
1596{
1597 bus_dma_tag_t dmat;
1598 struct bus_dmamap *map;
1599
1600 mtx_lock(&bounce_lock);
1601 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1602 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1603 mtx_unlock(&bounce_lock);
1604 dmat = map->dmat;
1605 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1606 bus_dmamap_load(map->dmat, map, map->buffer, map->len,
1607 map->callback, map->callback_arg, /*flags*/0);
1608 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1609 mtx_lock(&bounce_lock);
1610 }
1611 mtx_unlock(&bounce_lock);
1612}
|