29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/bus.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>
36#include <sys/ktr.h>
37#include <sys/lock.h>
38#include <sys/proc.h>
39#include <sys/memdesc.h>
40#include <sys/mutex.h>
41#include <sys/sysctl.h>
42#include <sys/uio.h>
43
44#include <vm/vm.h>
45#include <vm/vm_extern.h>
46#include <vm/vm_kern.h>
47#include <vm/vm_page.h>
48#include <vm/vm_map.h>
49
50#include <machine/atomic.h>
51#include <machine/bus.h>
52#include <machine/md_var.h>
53#include <machine/specialreg.h>
54
55#ifdef __i386__
56#define MAX_BPAGES 512
57#else
58#define MAX_BPAGES 8192
59#endif
60#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
61#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
62
63struct bounce_zone;
64
65struct bus_dma_tag {
66 bus_dma_tag_t parent;
67 bus_size_t alignment;
68 bus_addr_t boundary;
69 bus_addr_t lowaddr;
70 bus_addr_t highaddr;
71 bus_dma_filter_t *filter;
72 void *filterarg;
73 bus_size_t maxsize;
74 u_int nsegments;
75 bus_size_t maxsegsz;
76 int flags;
77 int ref_count;
78 int map_count;
79 bus_dma_lock_t *lockfunc;
80 void *lockfuncarg;
81 bus_dma_segment_t *segments;
82 struct bounce_zone *bounce_zone;
83};
84
85struct bounce_page {
86 vm_offset_t vaddr; /* kva of bounce buffer */
87 bus_addr_t busaddr; /* Physical address */
88 vm_offset_t datavaddr; /* kva of client data */
89 bus_addr_t dataaddr; /* client physical address */
90 bus_size_t datacount; /* client data count */
91 STAILQ_ENTRY(bounce_page) links;
92};
93
94int busdma_swi_pending;
95
96struct bounce_zone {
97 STAILQ_ENTRY(bounce_zone) links;
98 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
99 int total_bpages;
100 int free_bpages;
101 int reserved_bpages;
102 int active_bpages;
103 int total_bounced;
104 int total_deferred;
105 int map_count;
106 bus_size_t alignment;
107 bus_addr_t lowaddr;
108 char zoneid[8];
109 char lowaddrid[20];
110 struct sysctl_ctx_list sysctl_tree;
111 struct sysctl_oid *sysctl_tree_top;
112};
113
114static struct mtx bounce_lock;
115static int total_bpages;
116static int busdma_zonecount;
117static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
118
119static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
120SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
121 "Total bounce pages");
122
123struct bus_dmamap {
124 struct bp_list bpages;
125 int pagesneeded;
126 int pagesreserved;
127 bus_dma_tag_t dmat;
128 struct memdesc mem;
129 bus_dmamap_callback_t *callback;
130 void *callback_arg;
131 STAILQ_ENTRY(bus_dmamap) links;
132};
133
134static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
135static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
136static struct bus_dmamap nobounce_dmamap, contig_dmamap;
137
138static void init_bounce_pages(void *dummy);
139static int alloc_bounce_zone(bus_dma_tag_t dmat);
140static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
141static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
142 int commit);
143static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
144 vm_offset_t vaddr, bus_addr_t addr,
145 bus_size_t size);
146static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
147int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
148static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
149 pmap_t pmap, void *buf, bus_size_t buflen,
150 int flags);
151static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
152 vm_paddr_t buf, bus_size_t buflen,
153 int flags);
154static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
155 int flags);
156
157#ifdef XEN
158#undef pmap_kextract
159#define pmap_kextract pmap_kextract_ma
160#endif
161
162/*
163 * Return true if a match is made.
164 *
165 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
166 *
167 * If paddr is within the bounds of the dma tag then call the filter callback
168 * to check for a match, if there is no filter callback then assume a match.
169 */
170int
171run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
172{
173 int retval;
174
175 retval = 0;
176
177 do {
178 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
179 || ((paddr & (dmat->alignment - 1)) != 0))
180 && (dmat->filter == NULL
181 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
182 retval = 1;
183
184 dmat = dmat->parent;
185 } while (retval == 0 && dmat != NULL);
186 return (retval);
187}
188
189/*
190 * Convenience function for manipulating driver locks from busdma (during
191 * busdma_swi, for example). Drivers that don't provide their own locks
192 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
193 * non-mutex locking scheme don't have to use this at all.
194 */
195void
196busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
197{
198 struct mtx *dmtx;
199
200 dmtx = (struct mtx *)arg;
201 switch (op) {
202 case BUS_DMA_LOCK:
203 mtx_lock(dmtx);
204 break;
205 case BUS_DMA_UNLOCK:
206 mtx_unlock(dmtx);
207 break;
208 default:
209 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
210 }
211}
212
213/*
214 * dflt_lock should never get called. It gets put into the dma tag when
215 * lockfunc == NULL, which is only valid if the maps that are associated
216 * with the tag are meant to never be defered.
217 * XXX Should have a way to identify which driver is responsible here.
218 */
219static void
220dflt_lock(void *arg, bus_dma_lock_op_t op)
221{
222 panic("driver error: busdma dflt_lock called");
223}
224
225/*
226 * Allocate a device specific dma_tag.
227 */
228int
229bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
230 bus_addr_t boundary, bus_addr_t lowaddr,
231 bus_addr_t highaddr, bus_dma_filter_t *filter,
232 void *filterarg, bus_size_t maxsize, int nsegments,
233 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
234 void *lockfuncarg, bus_dma_tag_t *dmat)
235{
236 bus_dma_tag_t newtag;
237 int error = 0;
238
239 /* Basic sanity checking */
240 if (boundary != 0 && boundary < maxsegsz)
241 maxsegsz = boundary;
242
243 if (maxsegsz == 0) {
244 return (EINVAL);
245 }
246
247 /* Return a NULL tag on failure */
248 *dmat = NULL;
249
250 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
251 M_ZERO | M_NOWAIT);
252 if (newtag == NULL) {
253 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
254 __func__, newtag, 0, ENOMEM);
255 return (ENOMEM);
256 }
257
258 newtag->parent = parent;
259 newtag->alignment = alignment;
260 newtag->boundary = boundary;
261 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
262 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
263 newtag->filter = filter;
264 newtag->filterarg = filterarg;
265 newtag->maxsize = maxsize;
266 newtag->nsegments = nsegments;
267 newtag->maxsegsz = maxsegsz;
268 newtag->flags = flags;
269 newtag->ref_count = 1; /* Count ourself */
270 newtag->map_count = 0;
271 if (lockfunc != NULL) {
272 newtag->lockfunc = lockfunc;
273 newtag->lockfuncarg = lockfuncarg;
274 } else {
275 newtag->lockfunc = dflt_lock;
276 newtag->lockfuncarg = NULL;
277 }
278 newtag->segments = NULL;
279
280 /* Take into account any restrictions imposed by our parent tag */
281 if (parent != NULL) {
282 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
283 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
284 if (newtag->boundary == 0)
285 newtag->boundary = parent->boundary;
286 else if (parent->boundary != 0)
287 newtag->boundary = MIN(parent->boundary,
288 newtag->boundary);
289 if ((newtag->filter != NULL) ||
290 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
291 newtag->flags |= BUS_DMA_COULD_BOUNCE;
292 if (newtag->filter == NULL) {
293 /*
294 * Short circuit looking at our parent directly
295 * since we have encapsulated all of its information
296 */
297 newtag->filter = parent->filter;
298 newtag->filterarg = parent->filterarg;
299 newtag->parent = parent->parent;
300 }
301 if (newtag->parent != NULL)
302 atomic_add_int(&parent->ref_count, 1);
303 }
304
305 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
306 || newtag->alignment > 1)
307 newtag->flags |= BUS_DMA_COULD_BOUNCE;
308
309 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
310 (flags & BUS_DMA_ALLOCNOW) != 0) {
311 struct bounce_zone *bz;
312
313 /* Must bounce */
314
315 if ((error = alloc_bounce_zone(newtag)) != 0) {
316 free(newtag, M_DEVBUF);
317 return (error);
318 }
319 bz = newtag->bounce_zone;
320
321 if (ptoa(bz->total_bpages) < maxsize) {
322 int pages;
323
324 pages = atop(maxsize) - bz->total_bpages;
325
326 /* Add pages to our bounce pool */
327 if (alloc_bounce_pages(newtag, pages) < pages)
328 error = ENOMEM;
329 }
330 /* Performed initial allocation */
331 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
332 }
333
334 if (error != 0) {
335 free(newtag, M_DEVBUF);
336 } else {
337 *dmat = newtag;
338 }
339 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
340 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
341 return (error);
342}
343
344int
345bus_dma_tag_destroy(bus_dma_tag_t dmat)
346{
347 bus_dma_tag_t dmat_copy;
348 int error;
349
350 error = 0;
351 dmat_copy = dmat;
352
353 if (dmat != NULL) {
354
355 if (dmat->map_count != 0) {
356 error = EBUSY;
357 goto out;
358 }
359
360 while (dmat != NULL) {
361 bus_dma_tag_t parent;
362
363 parent = dmat->parent;
364 atomic_subtract_int(&dmat->ref_count, 1);
365 if (dmat->ref_count == 0) {
366 if (dmat->segments != NULL)
367 free(dmat->segments, M_DEVBUF);
368 free(dmat, M_DEVBUF);
369 /*
370 * Last reference count, so
371 * release our reference
372 * count on our parent.
373 */
374 dmat = parent;
375 } else
376 dmat = NULL;
377 }
378 }
379out:
380 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
381 return (error);
382}
383
384/*
385 * Allocate a handle for mapping from kva/uva/physical
386 * address space into bus device space.
387 */
388int
389bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
390{
391 int error;
392
393 error = 0;
394
395 if (dmat->segments == NULL) {
396 dmat->segments = (bus_dma_segment_t *)malloc(
397 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
398 M_NOWAIT);
399 if (dmat->segments == NULL) {
400 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
401 __func__, dmat, ENOMEM);
402 return (ENOMEM);
403 }
404 }
405
406 /*
407 * Bouncing might be required if the driver asks for an active
408 * exclusion region, a data alignment that is stricter than 1, and/or
409 * an active address boundary.
410 */
411 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
412
413 /* Must bounce */
414 struct bounce_zone *bz;
415 int maxpages;
416
417 if (dmat->bounce_zone == NULL) {
418 if ((error = alloc_bounce_zone(dmat)) != 0)
419 return (error);
420 }
421 bz = dmat->bounce_zone;
422
423 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
424 M_NOWAIT | M_ZERO);
425 if (*mapp == NULL) {
426 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
427 __func__, dmat, ENOMEM);
428 return (ENOMEM);
429 }
430
431 /* Initialize the new map */
432 STAILQ_INIT(&((*mapp)->bpages));
433
434 /*
435 * Attempt to add pages to our pool on a per-instance
436 * basis up to a sane limit.
437 */
438 if (dmat->alignment > 1)
439 maxpages = MAX_BPAGES;
440 else
441 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
442 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
443 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
444 int pages;
445
446 pages = MAX(atop(dmat->maxsize), 1);
447 pages = MIN(maxpages - bz->total_bpages, pages);
448 pages = MAX(pages, 1);
449 if (alloc_bounce_pages(dmat, pages) < pages)
450 error = ENOMEM;
451
452 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
453 if (error == 0)
454 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
455 } else {
456 error = 0;
457 }
458 }
459 bz->map_count++;
460 } else {
461 *mapp = NULL;
462 }
463 if (error == 0)
464 dmat->map_count++;
465 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
466 __func__, dmat, dmat->flags, error);
467 return (error);
468}
469
470/*
471 * Destroy a handle for mapping from kva/uva/physical
472 * address space into bus device space.
473 */
474int
475bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
476{
477 if (map != NULL && map != &nobounce_dmamap && map != &contig_dmamap) {
478 if (STAILQ_FIRST(&map->bpages) != NULL) {
479 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
480 __func__, dmat, EBUSY);
481 return (EBUSY);
482 }
483 if (dmat->bounce_zone)
484 dmat->bounce_zone->map_count--;
485 free(map, M_DEVBUF);
486 }
487 dmat->map_count--;
488 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
489 return (0);
490}
491
492
493/*
494 * Allocate a piece of memory that can be efficiently mapped into
495 * bus device space based on the constraints lited in the dma tag.
496 * A dmamap to for use with dmamap_load is also allocated.
497 */
498int
499bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
500 bus_dmamap_t *mapp)
501{
502 vm_memattr_t attr;
503 int mflags;
504
505 if (flags & BUS_DMA_NOWAIT)
506 mflags = M_NOWAIT;
507 else
508 mflags = M_WAITOK;
509
510 /* If we succeed, no mapping/bouncing will be required */
511 *mapp = NULL;
512
513 if (dmat->segments == NULL) {
514 dmat->segments = (bus_dma_segment_t *)malloc(
515 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
516 mflags);
517 if (dmat->segments == NULL) {
518 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
519 __func__, dmat, dmat->flags, ENOMEM);
520 return (ENOMEM);
521 }
522 }
523 if (flags & BUS_DMA_ZERO)
524 mflags |= M_ZERO;
525 if (flags & BUS_DMA_NOCACHE)
526 attr = VM_MEMATTR_UNCACHEABLE;
527 else
528 attr = VM_MEMATTR_DEFAULT;
529
530 /*
531 * XXX:
532 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
533 * alignment guarantees of malloc need to be nailed down, and the
534 * code below should be rewritten to take that into account.
535 *
536 * In the meantime, we'll warn the user if malloc gets it wrong.
537 */
538 if ((dmat->maxsize <= PAGE_SIZE) &&
539 (dmat->alignment < dmat->maxsize) &&
540 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
541 attr == VM_MEMATTR_DEFAULT) {
542 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
543 } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
544 dmat->alignment <= PAGE_SIZE &&
545 (dmat->boundary == 0 || dmat->boundary >= dmat->lowaddr)) {
546 /* Page-based multi-segment allocations allowed */
547 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
548 mflags, 0ul, dmat->lowaddr, attr);
549 *mapp = &contig_dmamap;
550 } else {
551 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
552 mflags, 0ul, dmat->lowaddr, dmat->alignment ?
553 dmat->alignment : 1ul, dmat->boundary, attr);
554 *mapp = &contig_dmamap;
555 }
556 if (*vaddr == NULL) {
557 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
558 __func__, dmat, dmat->flags, ENOMEM);
559 return (ENOMEM);
560 } else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
561 printf("bus_dmamem_alloc failed to align memory properly.\n");
562 }
563 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
564 __func__, dmat, dmat->flags, 0);
565 return (0);
566}
567
568/*
569 * Free a piece of memory and it's allociated dmamap, that was allocated
570 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
571 */
572void
573bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
574{
575 /*
576 * dmamem does not need to be bounced, so the map should be
577 * NULL if malloc() was used and contig_dmamap if
578 * kmem_alloc_contig() was used.
579 */
580 if (!(map == NULL || map == &contig_dmamap))
581 panic("bus_dmamem_free: Invalid map freed\n");
582 if (map == NULL)
583 free(vaddr, M_DEVBUF);
584 else
585 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
586 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
587}
588
589static void
590_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
591 bus_size_t buflen, int flags)
592{
593 bus_addr_t curaddr;
594 bus_size_t sgsize;
595
596 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
597 /*
598 * Count the number of bounce pages
599 * needed in order to complete this transfer
600 */
601 curaddr = buf;
602 while (buflen != 0) {
603 sgsize = MIN(buflen, dmat->maxsegsz);
604 if (run_filter(dmat, curaddr)) {
605 sgsize = MIN(sgsize, PAGE_SIZE);
606 map->pagesneeded++;
607 }
608 curaddr += sgsize;
609 buflen -= sgsize;
610 }
611 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
612 }
613}
614
615static void
616_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
617 void *buf, bus_size_t buflen, int flags)
618{
619 vm_offset_t vaddr;
620 vm_offset_t vendaddr;
621 bus_addr_t paddr;
622
623 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
624 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
625 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
626 dmat->boundary, dmat->alignment);
627 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
628 map, &nobounce_dmamap, map->pagesneeded);
629 /*
630 * Count the number of bounce pages
631 * needed in order to complete this transfer
632 */
633 vaddr = (vm_offset_t)buf;
634 vendaddr = (vm_offset_t)buf + buflen;
635
636 while (vaddr < vendaddr) {
637 bus_size_t sg_len;
638
639 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
640 if (pmap == kernel_pmap)
641 paddr = pmap_kextract(vaddr);
642 else
643 paddr = pmap_extract(pmap, vaddr);
644 if (run_filter(dmat, paddr) != 0) {
645 sg_len = roundup2(sg_len, dmat->alignment);
646 map->pagesneeded++;
647 }
648 vaddr += sg_len;
649 }
650 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
651 }
652}
653
654static int
655_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
656{
657
658 /* Reserve Necessary Bounce Pages */
659 mtx_lock(&bounce_lock);
660 if (flags & BUS_DMA_NOWAIT) {
661 if (reserve_bounce_pages(dmat, map, 0) != 0) {
662 mtx_unlock(&bounce_lock);
663 return (ENOMEM);
664 }
665 } else {
666 if (reserve_bounce_pages(dmat, map, 1) != 0) {
667 /* Queue us for resources */
668 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
669 mtx_unlock(&bounce_lock);
670 return (EINPROGRESS);
671 }
672 }
673 mtx_unlock(&bounce_lock);
674
675 return (0);
676}
677
678/*
679 * Add a single contiguous physical range to the segment list.
680 */
681static int
682_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
683 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
684{
685 bus_addr_t baddr, bmask;
686 int seg;
687
688 /*
689 * Make sure we don't cross any boundaries.
690 */
691 bmask = ~(dmat->boundary - 1);
692 if (dmat->boundary > 0) {
693 baddr = (curaddr + dmat->boundary) & bmask;
694 if (sgsize > (baddr - curaddr))
695 sgsize = (baddr - curaddr);
696 }
697
698 /*
699 * Insert chunk into a segment, coalescing with
700 * previous segment if possible.
701 */
702 seg = *segp;
703 if (seg == -1) {
704 seg = 0;
705 segs[seg].ds_addr = curaddr;
706 segs[seg].ds_len = sgsize;
707 } else {
708 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
709 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
710 (dmat->boundary == 0 ||
711 (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
712 segs[seg].ds_len += sgsize;
713 else {
714 if (++seg >= dmat->nsegments)
715 return (0);
716 segs[seg].ds_addr = curaddr;
717 segs[seg].ds_len = sgsize;
718 }
719 }
720 *segp = seg;
721 return (sgsize);
722}
723
724/*
725 * Utility function to load a physical buffer. segp contains
726 * the starting segment on entrace, and the ending segment on exit.
727 */
728int
729_bus_dmamap_load_phys(bus_dma_tag_t dmat,
730 bus_dmamap_t map,
731 vm_paddr_t buf, bus_size_t buflen,
732 int flags,
733 bus_dma_segment_t *segs,
734 int *segp)
735{
736 bus_size_t sgsize;
737 bus_addr_t curaddr;
738 int error;
739
740 if (map == NULL || map == &contig_dmamap)
741 map = &nobounce_dmamap;
742
743 if (segs == NULL)
744 segs = dmat->segments;
745
746 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
747 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
748 if (map->pagesneeded != 0) {
749 error = _bus_dmamap_reserve_pages(dmat, map, flags);
750 if (error)
751 return (error);
752 }
753 }
754
755 while (buflen > 0) {
756 curaddr = buf;
757 sgsize = MIN(buflen, dmat->maxsegsz);
758 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
759 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
760 sgsize = MIN(sgsize, PAGE_SIZE);
761 curaddr = add_bounce_page(dmat, map, 0, curaddr,
762 sgsize);
763 }
764 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
765 segp);
766 if (sgsize == 0)
767 break;
768 buf += sgsize;
769 buflen -= sgsize;
770 }
771
772 /*
773 * Did we fit?
774 */
775 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
776}
777
778/*
779 * Utility function to load a linear buffer. segp contains
780 * the starting segment on entrace, and the ending segment on exit.
781 */
782int
783_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
784 bus_dmamap_t map,
785 void *buf, bus_size_t buflen,
786 pmap_t pmap,
787 int flags,
788 bus_dma_segment_t *segs,
789 int *segp)
790{
791 bus_size_t sgsize;
792 bus_addr_t curaddr;
793 vm_offset_t vaddr;
794 int error;
795
796 if (map == NULL || map == &contig_dmamap)
797 map = &nobounce_dmamap;
798
799 if (segs == NULL)
800 segs = dmat->segments;
801
802 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
803 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
804 if (map->pagesneeded != 0) {
805 error = _bus_dmamap_reserve_pages(dmat, map, flags);
806 if (error)
807 return (error);
808 }
809 }
810
811 vaddr = (vm_offset_t)buf;
812
813 while (buflen > 0) {
814 bus_size_t max_sgsize;
815
816 /*
817 * Get the physical address for this segment.
818 */
819 if (pmap == kernel_pmap)
820 curaddr = pmap_kextract(vaddr);
821 else
822 curaddr = pmap_extract(pmap, vaddr);
823
824 /*
825 * Compute the segment size, and adjust counts.
826 */
827 max_sgsize = MIN(buflen, dmat->maxsegsz);
828 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
829 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
830 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
831 sgsize = roundup2(sgsize, dmat->alignment);
832 sgsize = MIN(sgsize, max_sgsize);
833 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
834 sgsize);
835 } else {
836 sgsize = MIN(sgsize, max_sgsize);
837 }
838 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
839 segp);
840 if (sgsize == 0)
841 break;
842 vaddr += sgsize;
843 buflen -= sgsize;
844 }
845
846 /*
847 * Did we fit?
848 */
849 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
850}
851
| 29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/bus.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>
36#include <sys/ktr.h>
37#include <sys/lock.h>
38#include <sys/proc.h>
39#include <sys/memdesc.h>
40#include <sys/mutex.h>
41#include <sys/sysctl.h>
42#include <sys/uio.h>
43
44#include <vm/vm.h>
45#include <vm/vm_extern.h>
46#include <vm/vm_kern.h>
47#include <vm/vm_page.h>
48#include <vm/vm_map.h>
49
50#include <machine/atomic.h>
51#include <machine/bus.h>
52#include <machine/md_var.h>
53#include <machine/specialreg.h>
54
55#ifdef __i386__
56#define MAX_BPAGES 512
57#else
58#define MAX_BPAGES 8192
59#endif
60#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
61#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
62
63struct bounce_zone;
64
65struct bus_dma_tag {
66 bus_dma_tag_t parent;
67 bus_size_t alignment;
68 bus_addr_t boundary;
69 bus_addr_t lowaddr;
70 bus_addr_t highaddr;
71 bus_dma_filter_t *filter;
72 void *filterarg;
73 bus_size_t maxsize;
74 u_int nsegments;
75 bus_size_t maxsegsz;
76 int flags;
77 int ref_count;
78 int map_count;
79 bus_dma_lock_t *lockfunc;
80 void *lockfuncarg;
81 bus_dma_segment_t *segments;
82 struct bounce_zone *bounce_zone;
83};
84
85struct bounce_page {
86 vm_offset_t vaddr; /* kva of bounce buffer */
87 bus_addr_t busaddr; /* Physical address */
88 vm_offset_t datavaddr; /* kva of client data */
89 bus_addr_t dataaddr; /* client physical address */
90 bus_size_t datacount; /* client data count */
91 STAILQ_ENTRY(bounce_page) links;
92};
93
94int busdma_swi_pending;
95
96struct bounce_zone {
97 STAILQ_ENTRY(bounce_zone) links;
98 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
99 int total_bpages;
100 int free_bpages;
101 int reserved_bpages;
102 int active_bpages;
103 int total_bounced;
104 int total_deferred;
105 int map_count;
106 bus_size_t alignment;
107 bus_addr_t lowaddr;
108 char zoneid[8];
109 char lowaddrid[20];
110 struct sysctl_ctx_list sysctl_tree;
111 struct sysctl_oid *sysctl_tree_top;
112};
113
114static struct mtx bounce_lock;
115static int total_bpages;
116static int busdma_zonecount;
117static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
118
119static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
120SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
121 "Total bounce pages");
122
123struct bus_dmamap {
124 struct bp_list bpages;
125 int pagesneeded;
126 int pagesreserved;
127 bus_dma_tag_t dmat;
128 struct memdesc mem;
129 bus_dmamap_callback_t *callback;
130 void *callback_arg;
131 STAILQ_ENTRY(bus_dmamap) links;
132};
133
134static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
135static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
136static struct bus_dmamap nobounce_dmamap, contig_dmamap;
137
138static void init_bounce_pages(void *dummy);
139static int alloc_bounce_zone(bus_dma_tag_t dmat);
140static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
141static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
142 int commit);
143static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
144 vm_offset_t vaddr, bus_addr_t addr,
145 bus_size_t size);
146static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
147int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
148static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
149 pmap_t pmap, void *buf, bus_size_t buflen,
150 int flags);
151static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
152 vm_paddr_t buf, bus_size_t buflen,
153 int flags);
154static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
155 int flags);
156
157#ifdef XEN
158#undef pmap_kextract
159#define pmap_kextract pmap_kextract_ma
160#endif
161
162/*
163 * Return true if a match is made.
164 *
165 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
166 *
167 * If paddr is within the bounds of the dma tag then call the filter callback
168 * to check for a match, if there is no filter callback then assume a match.
169 */
170int
171run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
172{
173 int retval;
174
175 retval = 0;
176
177 do {
178 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
179 || ((paddr & (dmat->alignment - 1)) != 0))
180 && (dmat->filter == NULL
181 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
182 retval = 1;
183
184 dmat = dmat->parent;
185 } while (retval == 0 && dmat != NULL);
186 return (retval);
187}
188
189/*
190 * Convenience function for manipulating driver locks from busdma (during
191 * busdma_swi, for example). Drivers that don't provide their own locks
192 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
193 * non-mutex locking scheme don't have to use this at all.
194 */
195void
196busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
197{
198 struct mtx *dmtx;
199
200 dmtx = (struct mtx *)arg;
201 switch (op) {
202 case BUS_DMA_LOCK:
203 mtx_lock(dmtx);
204 break;
205 case BUS_DMA_UNLOCK:
206 mtx_unlock(dmtx);
207 break;
208 default:
209 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
210 }
211}
212
213/*
214 * dflt_lock should never get called. It gets put into the dma tag when
215 * lockfunc == NULL, which is only valid if the maps that are associated
216 * with the tag are meant to never be defered.
217 * XXX Should have a way to identify which driver is responsible here.
218 */
219static void
220dflt_lock(void *arg, bus_dma_lock_op_t op)
221{
222 panic("driver error: busdma dflt_lock called");
223}
224
225/*
226 * Allocate a device specific dma_tag.
227 */
228int
229bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
230 bus_addr_t boundary, bus_addr_t lowaddr,
231 bus_addr_t highaddr, bus_dma_filter_t *filter,
232 void *filterarg, bus_size_t maxsize, int nsegments,
233 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
234 void *lockfuncarg, bus_dma_tag_t *dmat)
235{
236 bus_dma_tag_t newtag;
237 int error = 0;
238
239 /* Basic sanity checking */
240 if (boundary != 0 && boundary < maxsegsz)
241 maxsegsz = boundary;
242
243 if (maxsegsz == 0) {
244 return (EINVAL);
245 }
246
247 /* Return a NULL tag on failure */
248 *dmat = NULL;
249
250 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
251 M_ZERO | M_NOWAIT);
252 if (newtag == NULL) {
253 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
254 __func__, newtag, 0, ENOMEM);
255 return (ENOMEM);
256 }
257
258 newtag->parent = parent;
259 newtag->alignment = alignment;
260 newtag->boundary = boundary;
261 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
262 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
263 newtag->filter = filter;
264 newtag->filterarg = filterarg;
265 newtag->maxsize = maxsize;
266 newtag->nsegments = nsegments;
267 newtag->maxsegsz = maxsegsz;
268 newtag->flags = flags;
269 newtag->ref_count = 1; /* Count ourself */
270 newtag->map_count = 0;
271 if (lockfunc != NULL) {
272 newtag->lockfunc = lockfunc;
273 newtag->lockfuncarg = lockfuncarg;
274 } else {
275 newtag->lockfunc = dflt_lock;
276 newtag->lockfuncarg = NULL;
277 }
278 newtag->segments = NULL;
279
280 /* Take into account any restrictions imposed by our parent tag */
281 if (parent != NULL) {
282 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
283 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
284 if (newtag->boundary == 0)
285 newtag->boundary = parent->boundary;
286 else if (parent->boundary != 0)
287 newtag->boundary = MIN(parent->boundary,
288 newtag->boundary);
289 if ((newtag->filter != NULL) ||
290 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
291 newtag->flags |= BUS_DMA_COULD_BOUNCE;
292 if (newtag->filter == NULL) {
293 /*
294 * Short circuit looking at our parent directly
295 * since we have encapsulated all of its information
296 */
297 newtag->filter = parent->filter;
298 newtag->filterarg = parent->filterarg;
299 newtag->parent = parent->parent;
300 }
301 if (newtag->parent != NULL)
302 atomic_add_int(&parent->ref_count, 1);
303 }
304
305 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
306 || newtag->alignment > 1)
307 newtag->flags |= BUS_DMA_COULD_BOUNCE;
308
309 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
310 (flags & BUS_DMA_ALLOCNOW) != 0) {
311 struct bounce_zone *bz;
312
313 /* Must bounce */
314
315 if ((error = alloc_bounce_zone(newtag)) != 0) {
316 free(newtag, M_DEVBUF);
317 return (error);
318 }
319 bz = newtag->bounce_zone;
320
321 if (ptoa(bz->total_bpages) < maxsize) {
322 int pages;
323
324 pages = atop(maxsize) - bz->total_bpages;
325
326 /* Add pages to our bounce pool */
327 if (alloc_bounce_pages(newtag, pages) < pages)
328 error = ENOMEM;
329 }
330 /* Performed initial allocation */
331 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
332 }
333
334 if (error != 0) {
335 free(newtag, M_DEVBUF);
336 } else {
337 *dmat = newtag;
338 }
339 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
340 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
341 return (error);
342}
343
344int
345bus_dma_tag_destroy(bus_dma_tag_t dmat)
346{
347 bus_dma_tag_t dmat_copy;
348 int error;
349
350 error = 0;
351 dmat_copy = dmat;
352
353 if (dmat != NULL) {
354
355 if (dmat->map_count != 0) {
356 error = EBUSY;
357 goto out;
358 }
359
360 while (dmat != NULL) {
361 bus_dma_tag_t parent;
362
363 parent = dmat->parent;
364 atomic_subtract_int(&dmat->ref_count, 1);
365 if (dmat->ref_count == 0) {
366 if (dmat->segments != NULL)
367 free(dmat->segments, M_DEVBUF);
368 free(dmat, M_DEVBUF);
369 /*
370 * Last reference count, so
371 * release our reference
372 * count on our parent.
373 */
374 dmat = parent;
375 } else
376 dmat = NULL;
377 }
378 }
379out:
380 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
381 return (error);
382}
383
384/*
385 * Allocate a handle for mapping from kva/uva/physical
386 * address space into bus device space.
387 */
388int
389bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
390{
391 int error;
392
393 error = 0;
394
395 if (dmat->segments == NULL) {
396 dmat->segments = (bus_dma_segment_t *)malloc(
397 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
398 M_NOWAIT);
399 if (dmat->segments == NULL) {
400 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
401 __func__, dmat, ENOMEM);
402 return (ENOMEM);
403 }
404 }
405
406 /*
407 * Bouncing might be required if the driver asks for an active
408 * exclusion region, a data alignment that is stricter than 1, and/or
409 * an active address boundary.
410 */
411 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
412
413 /* Must bounce */
414 struct bounce_zone *bz;
415 int maxpages;
416
417 if (dmat->bounce_zone == NULL) {
418 if ((error = alloc_bounce_zone(dmat)) != 0)
419 return (error);
420 }
421 bz = dmat->bounce_zone;
422
423 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
424 M_NOWAIT | M_ZERO);
425 if (*mapp == NULL) {
426 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
427 __func__, dmat, ENOMEM);
428 return (ENOMEM);
429 }
430
431 /* Initialize the new map */
432 STAILQ_INIT(&((*mapp)->bpages));
433
434 /*
435 * Attempt to add pages to our pool on a per-instance
436 * basis up to a sane limit.
437 */
438 if (dmat->alignment > 1)
439 maxpages = MAX_BPAGES;
440 else
441 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
442 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
443 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
444 int pages;
445
446 pages = MAX(atop(dmat->maxsize), 1);
447 pages = MIN(maxpages - bz->total_bpages, pages);
448 pages = MAX(pages, 1);
449 if (alloc_bounce_pages(dmat, pages) < pages)
450 error = ENOMEM;
451
452 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
453 if (error == 0)
454 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
455 } else {
456 error = 0;
457 }
458 }
459 bz->map_count++;
460 } else {
461 *mapp = NULL;
462 }
463 if (error == 0)
464 dmat->map_count++;
465 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
466 __func__, dmat, dmat->flags, error);
467 return (error);
468}
469
470/*
471 * Destroy a handle for mapping from kva/uva/physical
472 * address space into bus device space.
473 */
474int
475bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
476{
477 if (map != NULL && map != &nobounce_dmamap && map != &contig_dmamap) {
478 if (STAILQ_FIRST(&map->bpages) != NULL) {
479 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
480 __func__, dmat, EBUSY);
481 return (EBUSY);
482 }
483 if (dmat->bounce_zone)
484 dmat->bounce_zone->map_count--;
485 free(map, M_DEVBUF);
486 }
487 dmat->map_count--;
488 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
489 return (0);
490}
491
492
493/*
494 * Allocate a piece of memory that can be efficiently mapped into
495 * bus device space based on the constraints lited in the dma tag.
496 * A dmamap to for use with dmamap_load is also allocated.
497 */
498int
499bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
500 bus_dmamap_t *mapp)
501{
502 vm_memattr_t attr;
503 int mflags;
504
505 if (flags & BUS_DMA_NOWAIT)
506 mflags = M_NOWAIT;
507 else
508 mflags = M_WAITOK;
509
510 /* If we succeed, no mapping/bouncing will be required */
511 *mapp = NULL;
512
513 if (dmat->segments == NULL) {
514 dmat->segments = (bus_dma_segment_t *)malloc(
515 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
516 mflags);
517 if (dmat->segments == NULL) {
518 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
519 __func__, dmat, dmat->flags, ENOMEM);
520 return (ENOMEM);
521 }
522 }
523 if (flags & BUS_DMA_ZERO)
524 mflags |= M_ZERO;
525 if (flags & BUS_DMA_NOCACHE)
526 attr = VM_MEMATTR_UNCACHEABLE;
527 else
528 attr = VM_MEMATTR_DEFAULT;
529
530 /*
531 * XXX:
532 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
533 * alignment guarantees of malloc need to be nailed down, and the
534 * code below should be rewritten to take that into account.
535 *
536 * In the meantime, we'll warn the user if malloc gets it wrong.
537 */
538 if ((dmat->maxsize <= PAGE_SIZE) &&
539 (dmat->alignment < dmat->maxsize) &&
540 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
541 attr == VM_MEMATTR_DEFAULT) {
542 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
543 } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
544 dmat->alignment <= PAGE_SIZE &&
545 (dmat->boundary == 0 || dmat->boundary >= dmat->lowaddr)) {
546 /* Page-based multi-segment allocations allowed */
547 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
548 mflags, 0ul, dmat->lowaddr, attr);
549 *mapp = &contig_dmamap;
550 } else {
551 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
552 mflags, 0ul, dmat->lowaddr, dmat->alignment ?
553 dmat->alignment : 1ul, dmat->boundary, attr);
554 *mapp = &contig_dmamap;
555 }
556 if (*vaddr == NULL) {
557 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
558 __func__, dmat, dmat->flags, ENOMEM);
559 return (ENOMEM);
560 } else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
561 printf("bus_dmamem_alloc failed to align memory properly.\n");
562 }
563 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
564 __func__, dmat, dmat->flags, 0);
565 return (0);
566}
567
568/*
569 * Free a piece of memory and it's allociated dmamap, that was allocated
570 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
571 */
572void
573bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
574{
575 /*
576 * dmamem does not need to be bounced, so the map should be
577 * NULL if malloc() was used and contig_dmamap if
578 * kmem_alloc_contig() was used.
579 */
580 if (!(map == NULL || map == &contig_dmamap))
581 panic("bus_dmamem_free: Invalid map freed\n");
582 if (map == NULL)
583 free(vaddr, M_DEVBUF);
584 else
585 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
586 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
587}
588
589static void
590_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
591 bus_size_t buflen, int flags)
592{
593 bus_addr_t curaddr;
594 bus_size_t sgsize;
595
596 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
597 /*
598 * Count the number of bounce pages
599 * needed in order to complete this transfer
600 */
601 curaddr = buf;
602 while (buflen != 0) {
603 sgsize = MIN(buflen, dmat->maxsegsz);
604 if (run_filter(dmat, curaddr)) {
605 sgsize = MIN(sgsize, PAGE_SIZE);
606 map->pagesneeded++;
607 }
608 curaddr += sgsize;
609 buflen -= sgsize;
610 }
611 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
612 }
613}
614
615static void
616_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
617 void *buf, bus_size_t buflen, int flags)
618{
619 vm_offset_t vaddr;
620 vm_offset_t vendaddr;
621 bus_addr_t paddr;
622
623 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
624 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
625 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
626 dmat->boundary, dmat->alignment);
627 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
628 map, &nobounce_dmamap, map->pagesneeded);
629 /*
630 * Count the number of bounce pages
631 * needed in order to complete this transfer
632 */
633 vaddr = (vm_offset_t)buf;
634 vendaddr = (vm_offset_t)buf + buflen;
635
636 while (vaddr < vendaddr) {
637 bus_size_t sg_len;
638
639 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
640 if (pmap == kernel_pmap)
641 paddr = pmap_kextract(vaddr);
642 else
643 paddr = pmap_extract(pmap, vaddr);
644 if (run_filter(dmat, paddr) != 0) {
645 sg_len = roundup2(sg_len, dmat->alignment);
646 map->pagesneeded++;
647 }
648 vaddr += sg_len;
649 }
650 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
651 }
652}
653
654static int
655_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
656{
657
658 /* Reserve Necessary Bounce Pages */
659 mtx_lock(&bounce_lock);
660 if (flags & BUS_DMA_NOWAIT) {
661 if (reserve_bounce_pages(dmat, map, 0) != 0) {
662 mtx_unlock(&bounce_lock);
663 return (ENOMEM);
664 }
665 } else {
666 if (reserve_bounce_pages(dmat, map, 1) != 0) {
667 /* Queue us for resources */
668 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
669 mtx_unlock(&bounce_lock);
670 return (EINPROGRESS);
671 }
672 }
673 mtx_unlock(&bounce_lock);
674
675 return (0);
676}
677
678/*
679 * Add a single contiguous physical range to the segment list.
680 */
681static int
682_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
683 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
684{
685 bus_addr_t baddr, bmask;
686 int seg;
687
688 /*
689 * Make sure we don't cross any boundaries.
690 */
691 bmask = ~(dmat->boundary - 1);
692 if (dmat->boundary > 0) {
693 baddr = (curaddr + dmat->boundary) & bmask;
694 if (sgsize > (baddr - curaddr))
695 sgsize = (baddr - curaddr);
696 }
697
698 /*
699 * Insert chunk into a segment, coalescing with
700 * previous segment if possible.
701 */
702 seg = *segp;
703 if (seg == -1) {
704 seg = 0;
705 segs[seg].ds_addr = curaddr;
706 segs[seg].ds_len = sgsize;
707 } else {
708 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
709 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
710 (dmat->boundary == 0 ||
711 (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
712 segs[seg].ds_len += sgsize;
713 else {
714 if (++seg >= dmat->nsegments)
715 return (0);
716 segs[seg].ds_addr = curaddr;
717 segs[seg].ds_len = sgsize;
718 }
719 }
720 *segp = seg;
721 return (sgsize);
722}
723
724/*
725 * Utility function to load a physical buffer. segp contains
726 * the starting segment on entrace, and the ending segment on exit.
727 */
728int
729_bus_dmamap_load_phys(bus_dma_tag_t dmat,
730 bus_dmamap_t map,
731 vm_paddr_t buf, bus_size_t buflen,
732 int flags,
733 bus_dma_segment_t *segs,
734 int *segp)
735{
736 bus_size_t sgsize;
737 bus_addr_t curaddr;
738 int error;
739
740 if (map == NULL || map == &contig_dmamap)
741 map = &nobounce_dmamap;
742
743 if (segs == NULL)
744 segs = dmat->segments;
745
746 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
747 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
748 if (map->pagesneeded != 0) {
749 error = _bus_dmamap_reserve_pages(dmat, map, flags);
750 if (error)
751 return (error);
752 }
753 }
754
755 while (buflen > 0) {
756 curaddr = buf;
757 sgsize = MIN(buflen, dmat->maxsegsz);
758 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
759 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
760 sgsize = MIN(sgsize, PAGE_SIZE);
761 curaddr = add_bounce_page(dmat, map, 0, curaddr,
762 sgsize);
763 }
764 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
765 segp);
766 if (sgsize == 0)
767 break;
768 buf += sgsize;
769 buflen -= sgsize;
770 }
771
772 /*
773 * Did we fit?
774 */
775 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
776}
777
778/*
779 * Utility function to load a linear buffer. segp contains
780 * the starting segment on entrace, and the ending segment on exit.
781 */
782int
783_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
784 bus_dmamap_t map,
785 void *buf, bus_size_t buflen,
786 pmap_t pmap,
787 int flags,
788 bus_dma_segment_t *segs,
789 int *segp)
790{
791 bus_size_t sgsize;
792 bus_addr_t curaddr;
793 vm_offset_t vaddr;
794 int error;
795
796 if (map == NULL || map == &contig_dmamap)
797 map = &nobounce_dmamap;
798
799 if (segs == NULL)
800 segs = dmat->segments;
801
802 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
803 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
804 if (map->pagesneeded != 0) {
805 error = _bus_dmamap_reserve_pages(dmat, map, flags);
806 if (error)
807 return (error);
808 }
809 }
810
811 vaddr = (vm_offset_t)buf;
812
813 while (buflen > 0) {
814 bus_size_t max_sgsize;
815
816 /*
817 * Get the physical address for this segment.
818 */
819 if (pmap == kernel_pmap)
820 curaddr = pmap_kextract(vaddr);
821 else
822 curaddr = pmap_extract(pmap, vaddr);
823
824 /*
825 * Compute the segment size, and adjust counts.
826 */
827 max_sgsize = MIN(buflen, dmat->maxsegsz);
828 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
829 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
830 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
831 sgsize = roundup2(sgsize, dmat->alignment);
832 sgsize = MIN(sgsize, max_sgsize);
833 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
834 sgsize);
835 } else {
836 sgsize = MIN(sgsize, max_sgsize);
837 }
838 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
839 segp);
840 if (sgsize == 0)
841 break;
842 vaddr += sgsize;
843 buflen -= sgsize;
844 }
845
846 /*
847 * Did we fit?
848 */
849 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
850}
851
|
852void
853__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
854 struct memdesc *mem, bus_dmamap_callback_t *callback,
855 void *callback_arg)
856{
857 if (map != NULL) {
858 map->mem = *mem;
859 map->dmat = dmat;
860 map->callback = callback;
861 map->callback_arg = callback_arg;
862 }
863}
864
865bus_dma_segment_t *
866_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
867 bus_dma_segment_t *segs, int nsegs, int error)
868{
869
870 if (segs == NULL)
871 segs = dmat->segments;
872 return (segs);
873}
874
875/*
876 * Release the mapping held by map.
877 */
878void
879_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
880{
881 struct bounce_page *bpage;
882
883 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
884 STAILQ_REMOVE_HEAD(&map->bpages, links);
885 free_bounce_page(dmat, bpage);
886 }
887}
888
889void
890_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
891{
892 struct bounce_page *bpage;
893
894 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
895 /*
896 * Handle data bouncing. We might also
897 * want to add support for invalidating
898 * the caches on broken hardware
899 */
900 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
901 "performing bounce", __func__, dmat, dmat->flags, op);
902
903 if (op & BUS_DMASYNC_PREWRITE) {
904 while (bpage != NULL) {
905 if (bpage->datavaddr != 0)
906 bcopy((void *)bpage->datavaddr,
907 (void *)bpage->vaddr,
908 bpage->datacount);
909 else
910 physcopyout(bpage->dataaddr,
911 (void *)bpage->vaddr,
912 bpage->datacount);
913 bpage = STAILQ_NEXT(bpage, links);
914 }
915 dmat->bounce_zone->total_bounced++;
916 }
917
918 if (op & BUS_DMASYNC_POSTREAD) {
919 while (bpage != NULL) {
920 if (bpage->datavaddr != 0)
921 bcopy((void *)bpage->vaddr,
922 (void *)bpage->datavaddr,
923 bpage->datacount);
924 else
925 physcopyin((void *)bpage->vaddr,
926 bpage->dataaddr,
927 bpage->datacount);
928 bpage = STAILQ_NEXT(bpage, links);
929 }
930 dmat->bounce_zone->total_bounced++;
931 }
932 }
933}
934
935static void
936init_bounce_pages(void *dummy __unused)
937{
938
939 total_bpages = 0;
940 STAILQ_INIT(&bounce_zone_list);
941 STAILQ_INIT(&bounce_map_waitinglist);
942 STAILQ_INIT(&bounce_map_callbacklist);
943 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
944}
945SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
946
947static struct sysctl_ctx_list *
948busdma_sysctl_tree(struct bounce_zone *bz)
949{
950 return (&bz->sysctl_tree);
951}
952
953static struct sysctl_oid *
954busdma_sysctl_tree_top(struct bounce_zone *bz)
955{
956 return (bz->sysctl_tree_top);
957}
958
959#if defined(__amd64__) || defined(PAE)
960#define SYSCTL_ADD_BUS_SIZE_T SYSCTL_ADD_UQUAD
961#else
962#define SYSCTL_ADD_BUS_SIZE_T(ctx, parent, nbr, name, flag, ptr, desc) \
963 SYSCTL_ADD_UINT(ctx, parent, nbr, name, flag, ptr, 0, desc)
964#endif
965
966static int
967alloc_bounce_zone(bus_dma_tag_t dmat)
968{
969 struct bounce_zone *bz;
970
971 /* Check to see if we already have a suitable zone */
972 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
973 if ((dmat->alignment <= bz->alignment)
974 && (dmat->lowaddr >= bz->lowaddr)) {
975 dmat->bounce_zone = bz;
976 return (0);
977 }
978 }
979
980 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
981 M_NOWAIT | M_ZERO)) == NULL)
982 return (ENOMEM);
983
984 STAILQ_INIT(&bz->bounce_page_list);
985 bz->free_bpages = 0;
986 bz->reserved_bpages = 0;
987 bz->active_bpages = 0;
988 bz->lowaddr = dmat->lowaddr;
989 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
990 bz->map_count = 0;
991 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
992 busdma_zonecount++;
993 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
994 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
995 dmat->bounce_zone = bz;
996
997 sysctl_ctx_init(&bz->sysctl_tree);
998 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
999 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1000 CTLFLAG_RD, 0, "");
1001 if (bz->sysctl_tree_top == NULL) {
1002 sysctl_ctx_free(&bz->sysctl_tree);
1003 return (0); /* XXX error code? */
1004 }
1005
1006 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1007 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1008 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1009 "Total bounce pages");
1010 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1011 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1012 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1013 "Free bounce pages");
1014 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1015 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1016 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1017 "Reserved bounce pages");
1018 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1019 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1020 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1021 "Active bounce pages");
1022 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1023 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1024 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1025 "Total bounce requests");
1026 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1027 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1028 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1029 "Total bounce requests that were deferred");
1030 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1031 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1032 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1033 SYSCTL_ADD_BUS_SIZE_T(busdma_sysctl_tree(bz),
1034 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1035 "alignment", CTLFLAG_RD, &bz->alignment, "");
1036
1037 return (0);
1038}
1039
1040static int
1041alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1042{
1043 struct bounce_zone *bz;
1044 int count;
1045
1046 bz = dmat->bounce_zone;
1047 count = 0;
1048 while (numpages > 0) {
1049 struct bounce_page *bpage;
1050
1051 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1052 M_NOWAIT | M_ZERO);
1053
1054 if (bpage == NULL)
1055 break;
1056 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1057 M_NOWAIT, 0ul,
1058 bz->lowaddr,
1059 PAGE_SIZE,
1060 0);
1061 if (bpage->vaddr == 0) {
1062 free(bpage, M_DEVBUF);
1063 break;
1064 }
1065 bpage->busaddr = pmap_kextract(bpage->vaddr);
1066 mtx_lock(&bounce_lock);
1067 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1068 total_bpages++;
1069 bz->total_bpages++;
1070 bz->free_bpages++;
1071 mtx_unlock(&bounce_lock);
1072 count++;
1073 numpages--;
1074 }
1075 return (count);
1076}
1077
1078static int
1079reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1080{
1081 struct bounce_zone *bz;
1082 int pages;
1083
1084 mtx_assert(&bounce_lock, MA_OWNED);
1085 bz = dmat->bounce_zone;
1086 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1087 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1088 return (map->pagesneeded - (map->pagesreserved + pages));
1089 bz->free_bpages -= pages;
1090 bz->reserved_bpages += pages;
1091 map->pagesreserved += pages;
1092 pages = map->pagesneeded - map->pagesreserved;
1093
1094 return (pages);
1095}
1096
1097static bus_addr_t
1098add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1099 bus_addr_t addr, bus_size_t size)
1100{
1101 struct bounce_zone *bz;
1102 struct bounce_page *bpage;
1103
1104 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1105 KASSERT(map != NULL && map != &nobounce_dmamap && map != &contig_dmamap,
1106 ("add_bounce_page: bad map %p", map));
1107
1108 bz = dmat->bounce_zone;
1109 if (map->pagesneeded == 0)
1110 panic("add_bounce_page: map doesn't need any pages");
1111 map->pagesneeded--;
1112
1113 if (map->pagesreserved == 0)
1114 panic("add_bounce_page: map doesn't need any pages");
1115 map->pagesreserved--;
1116
1117 mtx_lock(&bounce_lock);
1118 bpage = STAILQ_FIRST(&bz->bounce_page_list);
1119 if (bpage == NULL)
1120 panic("add_bounce_page: free page list is empty");
1121
1122 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1123 bz->reserved_bpages--;
1124 bz->active_bpages++;
1125 mtx_unlock(&bounce_lock);
1126
1127 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1128 /* Page offset needs to be preserved. */
1129 bpage->vaddr |= vaddr & PAGE_MASK;
1130 bpage->busaddr |= vaddr & PAGE_MASK;
1131 }
1132 bpage->datavaddr = vaddr;
1133 bpage->dataaddr = addr;
1134 bpage->datacount = size;
1135 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1136 return (bpage->busaddr);
1137}
1138
1139static void
1140free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1141{
1142 struct bus_dmamap *map;
1143 struct bounce_zone *bz;
1144
1145 bz = dmat->bounce_zone;
1146 bpage->datavaddr = 0;
1147 bpage->datacount = 0;
1148 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1149 /*
1150 * Reset the bounce page to start at offset 0. Other uses
1151 * of this bounce page may need to store a full page of
1152 * data and/or assume it starts on a page boundary.
1153 */
1154 bpage->vaddr &= ~PAGE_MASK;
1155 bpage->busaddr &= ~PAGE_MASK;
1156 }
1157
1158 mtx_lock(&bounce_lock);
1159 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1160 bz->free_bpages++;
1161 bz->active_bpages--;
1162 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1163 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1164 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1165 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1166 map, links);
1167 busdma_swi_pending = 1;
1168 bz->total_deferred++;
1169 swi_sched(vm_ih, 0);
1170 }
1171 }
1172 mtx_unlock(&bounce_lock);
1173}
1174
1175void
1176busdma_swi(void)
1177{
1178 bus_dma_tag_t dmat;
1179 struct bus_dmamap *map;
1180
1181 mtx_lock(&bounce_lock);
1182 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1183 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1184 mtx_unlock(&bounce_lock);
1185 dmat = map->dmat;
1186 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1187 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1188 map->callback, map->callback_arg,
1189 BUS_DMA_WAITOK);
1190 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1191 mtx_lock(&bounce_lock);
1192 }
1193 mtx_unlock(&bounce_lock);
1194}
| 862void
863__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
864 struct memdesc *mem, bus_dmamap_callback_t *callback,
865 void *callback_arg)
866{
867 if (map != NULL) {
868 map->mem = *mem;
869 map->dmat = dmat;
870 map->callback = callback;
871 map->callback_arg = callback_arg;
872 }
873}
874
875bus_dma_segment_t *
876_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
877 bus_dma_segment_t *segs, int nsegs, int error)
878{
879
880 if (segs == NULL)
881 segs = dmat->segments;
882 return (segs);
883}
884
885/*
886 * Release the mapping held by map.
887 */
888void
889_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
890{
891 struct bounce_page *bpage;
892
893 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
894 STAILQ_REMOVE_HEAD(&map->bpages, links);
895 free_bounce_page(dmat, bpage);
896 }
897}
898
899void
900_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
901{
902 struct bounce_page *bpage;
903
904 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
905 /*
906 * Handle data bouncing. We might also
907 * want to add support for invalidating
908 * the caches on broken hardware
909 */
910 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
911 "performing bounce", __func__, dmat, dmat->flags, op);
912
913 if (op & BUS_DMASYNC_PREWRITE) {
914 while (bpage != NULL) {
915 if (bpage->datavaddr != 0)
916 bcopy((void *)bpage->datavaddr,
917 (void *)bpage->vaddr,
918 bpage->datacount);
919 else
920 physcopyout(bpage->dataaddr,
921 (void *)bpage->vaddr,
922 bpage->datacount);
923 bpage = STAILQ_NEXT(bpage, links);
924 }
925 dmat->bounce_zone->total_bounced++;
926 }
927
928 if (op & BUS_DMASYNC_POSTREAD) {
929 while (bpage != NULL) {
930 if (bpage->datavaddr != 0)
931 bcopy((void *)bpage->vaddr,
932 (void *)bpage->datavaddr,
933 bpage->datacount);
934 else
935 physcopyin((void *)bpage->vaddr,
936 bpage->dataaddr,
937 bpage->datacount);
938 bpage = STAILQ_NEXT(bpage, links);
939 }
940 dmat->bounce_zone->total_bounced++;
941 }
942 }
943}
944
945static void
946init_bounce_pages(void *dummy __unused)
947{
948
949 total_bpages = 0;
950 STAILQ_INIT(&bounce_zone_list);
951 STAILQ_INIT(&bounce_map_waitinglist);
952 STAILQ_INIT(&bounce_map_callbacklist);
953 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
954}
955SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
956
957static struct sysctl_ctx_list *
958busdma_sysctl_tree(struct bounce_zone *bz)
959{
960 return (&bz->sysctl_tree);
961}
962
963static struct sysctl_oid *
964busdma_sysctl_tree_top(struct bounce_zone *bz)
965{
966 return (bz->sysctl_tree_top);
967}
968
969#if defined(__amd64__) || defined(PAE)
970#define SYSCTL_ADD_BUS_SIZE_T SYSCTL_ADD_UQUAD
971#else
972#define SYSCTL_ADD_BUS_SIZE_T(ctx, parent, nbr, name, flag, ptr, desc) \
973 SYSCTL_ADD_UINT(ctx, parent, nbr, name, flag, ptr, 0, desc)
974#endif
975
976static int
977alloc_bounce_zone(bus_dma_tag_t dmat)
978{
979 struct bounce_zone *bz;
980
981 /* Check to see if we already have a suitable zone */
982 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
983 if ((dmat->alignment <= bz->alignment)
984 && (dmat->lowaddr >= bz->lowaddr)) {
985 dmat->bounce_zone = bz;
986 return (0);
987 }
988 }
989
990 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
991 M_NOWAIT | M_ZERO)) == NULL)
992 return (ENOMEM);
993
994 STAILQ_INIT(&bz->bounce_page_list);
995 bz->free_bpages = 0;
996 bz->reserved_bpages = 0;
997 bz->active_bpages = 0;
998 bz->lowaddr = dmat->lowaddr;
999 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1000 bz->map_count = 0;
1001 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1002 busdma_zonecount++;
1003 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1004 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1005 dmat->bounce_zone = bz;
1006
1007 sysctl_ctx_init(&bz->sysctl_tree);
1008 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1009 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1010 CTLFLAG_RD, 0, "");
1011 if (bz->sysctl_tree_top == NULL) {
1012 sysctl_ctx_free(&bz->sysctl_tree);
1013 return (0); /* XXX error code? */
1014 }
1015
1016 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1017 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1018 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1019 "Total bounce pages");
1020 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1021 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1022 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1023 "Free bounce pages");
1024 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1025 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1026 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1027 "Reserved bounce pages");
1028 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1029 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1030 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1031 "Active bounce pages");
1032 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1033 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1034 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1035 "Total bounce requests");
1036 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1037 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1038 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1039 "Total bounce requests that were deferred");
1040 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1041 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1042 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1043 SYSCTL_ADD_BUS_SIZE_T(busdma_sysctl_tree(bz),
1044 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1045 "alignment", CTLFLAG_RD, &bz->alignment, "");
1046
1047 return (0);
1048}
1049
1050static int
1051alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1052{
1053 struct bounce_zone *bz;
1054 int count;
1055
1056 bz = dmat->bounce_zone;
1057 count = 0;
1058 while (numpages > 0) {
1059 struct bounce_page *bpage;
1060
1061 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1062 M_NOWAIT | M_ZERO);
1063
1064 if (bpage == NULL)
1065 break;
1066 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1067 M_NOWAIT, 0ul,
1068 bz->lowaddr,
1069 PAGE_SIZE,
1070 0);
1071 if (bpage->vaddr == 0) {
1072 free(bpage, M_DEVBUF);
1073 break;
1074 }
1075 bpage->busaddr = pmap_kextract(bpage->vaddr);
1076 mtx_lock(&bounce_lock);
1077 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1078 total_bpages++;
1079 bz->total_bpages++;
1080 bz->free_bpages++;
1081 mtx_unlock(&bounce_lock);
1082 count++;
1083 numpages--;
1084 }
1085 return (count);
1086}
1087
1088static int
1089reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1090{
1091 struct bounce_zone *bz;
1092 int pages;
1093
1094 mtx_assert(&bounce_lock, MA_OWNED);
1095 bz = dmat->bounce_zone;
1096 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1097 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1098 return (map->pagesneeded - (map->pagesreserved + pages));
1099 bz->free_bpages -= pages;
1100 bz->reserved_bpages += pages;
1101 map->pagesreserved += pages;
1102 pages = map->pagesneeded - map->pagesreserved;
1103
1104 return (pages);
1105}
1106
1107static bus_addr_t
1108add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1109 bus_addr_t addr, bus_size_t size)
1110{
1111 struct bounce_zone *bz;
1112 struct bounce_page *bpage;
1113
1114 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1115 KASSERT(map != NULL && map != &nobounce_dmamap && map != &contig_dmamap,
1116 ("add_bounce_page: bad map %p", map));
1117
1118 bz = dmat->bounce_zone;
1119 if (map->pagesneeded == 0)
1120 panic("add_bounce_page: map doesn't need any pages");
1121 map->pagesneeded--;
1122
1123 if (map->pagesreserved == 0)
1124 panic("add_bounce_page: map doesn't need any pages");
1125 map->pagesreserved--;
1126
1127 mtx_lock(&bounce_lock);
1128 bpage = STAILQ_FIRST(&bz->bounce_page_list);
1129 if (bpage == NULL)
1130 panic("add_bounce_page: free page list is empty");
1131
1132 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1133 bz->reserved_bpages--;
1134 bz->active_bpages++;
1135 mtx_unlock(&bounce_lock);
1136
1137 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1138 /* Page offset needs to be preserved. */
1139 bpage->vaddr |= vaddr & PAGE_MASK;
1140 bpage->busaddr |= vaddr & PAGE_MASK;
1141 }
1142 bpage->datavaddr = vaddr;
1143 bpage->dataaddr = addr;
1144 bpage->datacount = size;
1145 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1146 return (bpage->busaddr);
1147}
1148
1149static void
1150free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1151{
1152 struct bus_dmamap *map;
1153 struct bounce_zone *bz;
1154
1155 bz = dmat->bounce_zone;
1156 bpage->datavaddr = 0;
1157 bpage->datacount = 0;
1158 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1159 /*
1160 * Reset the bounce page to start at offset 0. Other uses
1161 * of this bounce page may need to store a full page of
1162 * data and/or assume it starts on a page boundary.
1163 */
1164 bpage->vaddr &= ~PAGE_MASK;
1165 bpage->busaddr &= ~PAGE_MASK;
1166 }
1167
1168 mtx_lock(&bounce_lock);
1169 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1170 bz->free_bpages++;
1171 bz->active_bpages--;
1172 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1173 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1174 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1175 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1176 map, links);
1177 busdma_swi_pending = 1;
1178 bz->total_deferred++;
1179 swi_sched(vm_ih, 0);
1180 }
1181 }
1182 mtx_unlock(&bounce_lock);
1183}
1184
1185void
1186busdma_swi(void)
1187{
1188 bus_dma_tag_t dmat;
1189 struct bus_dmamap *map;
1190
1191 mtx_lock(&bounce_lock);
1192 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1193 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1194 mtx_unlock(&bounce_lock);
1195 dmat = map->dmat;
1196 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1197 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1198 map->callback, map->callback_arg,
1199 BUS_DMA_WAITOK);
1200 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1201 mtx_lock(&bounce_lock);
1202 }
1203 mtx_unlock(&bounce_lock);
1204}
|