Deleted Added
sdiff udiff text old ( 289862 ) new ( 289864 )
full compact
busdma_machdep-v4.c (289862) busdma_machdep-v4.c (289864)
1/*-
2 * Copyright (c) 2012 Ian Lepore
3 * Copyright (c) 2004 Olivier Houchard
4 * Copyright (c) 2002 Peter Grehan
5 * Copyright (c) 1997, 1998 Justin T. Gibbs.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 16 unchanged lines hidden (view full) ---

25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2012 Ian Lepore
3 * Copyright (c) 2004 Olivier Houchard
4 * Copyright (c) 2002 Peter Grehan
5 * Copyright (c) 1997, 1998 Justin T. Gibbs.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 16 unchanged lines hidden (view full) ---

25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289862 2015-10-24 02:18:14Z ian $");
33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289864 2015-10-24 02:44:13Z ian $");
34
35/*
36 * ARM bus dma support routines.
37 *
38 * XXX Things to investigate / fix some day...
39 * - What is the earliest that this API can be called? Could there be any
40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the

--- 11 unchanged lines hidden (view full) ---

53 */
54
55#define _ARM32_BUS_DMA_PRIVATE
56#include <sys/param.h>
57#include <sys/systm.h>
58#include <sys/malloc.h>
59#include <sys/bus.h>
60#include <sys/busdma_bufalloc.h>
34
35/*
36 * ARM bus dma support routines.
37 *
38 * XXX Things to investigate / fix some day...
39 * - What is the earliest that this API can be called? Could there be any
40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM?
41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the

--- 11 unchanged lines hidden (view full) ---

53 */
54
55#define _ARM32_BUS_DMA_PRIVATE
56#include <sys/param.h>
57#include <sys/systm.h>
58#include <sys/malloc.h>
59#include <sys/bus.h>
60#include <sys/busdma_bufalloc.h>
61#include <sys/counter.h>
61#include <sys/interrupt.h>
62#include <sys/interrupt.h>
63#include <sys/kernel.h>
64#include <sys/ktr.h>
62#include <sys/lock.h>
65#include <sys/lock.h>
63#include <sys/proc.h>
64#include <sys/memdesc.h>
66#include <sys/memdesc.h>
67#include <sys/proc.h>
65#include <sys/mutex.h>
68#include <sys/mutex.h>
66#include <sys/ktr.h>
67#include <sys/kernel.h>
68#include <sys/sysctl.h>
69#include <sys/uio.h>
70
69#include <sys/sysctl.h>
70#include <sys/uio.h>
71
71#include <vm/uma.h>
72#include <vm/vm.h>
72#include <vm/vm.h>
73#include <vm/vm_extern.h>
74#include <vm/vm_kern.h>
75#include <vm/vm_page.h>
76#include <vm/vm_map.h>
73#include <vm/vm_page.h>
74#include <vm/vm_map.h>
75#include <vm/vm_extern.h>
76#include <vm/vm_kern.h>
77
78#include <machine/atomic.h>
79#include <machine/bus.h>
80#include <machine/cpufunc.h>
81#include <machine/md_var.h>
82
83#define MAX_BPAGES 64
84#define MAX_DMA_SEGMENTS 4096

--- 64 unchanged lines hidden (view full) ---

149 char lowaddrid[20];
150 struct sysctl_ctx_list sysctl_tree;
151 struct sysctl_oid *sysctl_tree_top;
152};
153
154static struct mtx bounce_lock;
155static int total_bpages;
156static int busdma_zonecount;
77
78#include <machine/atomic.h>
79#include <machine/bus.h>
80#include <machine/cpufunc.h>
81#include <machine/md_var.h>
82
83#define MAX_BPAGES 64
84#define MAX_DMA_SEGMENTS 4096

--- 64 unchanged lines hidden (view full) ---

149 char lowaddrid[20];
150 struct sysctl_ctx_list sysctl_tree;
151 struct sysctl_oid *sysctl_tree_top;
152};
153
154static struct mtx bounce_lock;
155static int total_bpages;
156static int busdma_zonecount;
157static uint32_t tags_total;
158static uint32_t maps_total;
159static uint32_t maps_dmamem;
160static uint32_t maps_coherent;
161static counter_u64_t maploads_total;
162static counter_u64_t maploads_bounced;
163static counter_u64_t maploads_coherent;
164static counter_u64_t maploads_dmamem;
165static counter_u64_t maploads_mbuf;
166static counter_u64_t maploads_physmem;
167
157static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
158
168static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
169
159static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
170SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
171SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0,
172 "Number of active tags");
173SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0,
174 "Number of active maps");
175SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0,
176 "Number of active maps for bus_dmamem_alloc buffers");
177SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0,
178 "Number of active maps with BUS_DMA_COHERENT flag set");
179SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD,
180 &maploads_total, "Number of load operations performed");
181SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD,
182 &maploads_bounced, "Number of load operations that used bounce buffers");
183SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD,
184 &maploads_dmamem, "Number of load operations on BUS_DMA_COHERENT memory");
185SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD,
186 &maploads_dmamem, "Number of load operations on bus_dmamem_alloc buffers");
187SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD,
188 &maploads_mbuf, "Number of load operations for mbufs");
189SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD,
190 &maploads_physmem, "Number of load operations on physical buffers");
160SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
191SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
161 "Total bounce pages");
192 "Total bounce pages");
162
163struct bus_dmamap {
164 struct bp_list bpages;
165 int pagesneeded;
166 int pagesreserved;
167 bus_dma_tag_t dmat;
168 struct memdesc mem;
169 bus_dmamap_callback_t *callback;

--- 36 unchanged lines hidden (view full) ---

206
207MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
208MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages");
209
210static void
211busdma_init(void *dummy)
212{
213
193
194struct bus_dmamap {
195 struct bp_list bpages;
196 int pagesneeded;
197 int pagesreserved;
198 bus_dma_tag_t dmat;
199 struct memdesc mem;
200 bus_dmamap_callback_t *callback;

--- 36 unchanged lines hidden (view full) ---

237
238MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
239MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages");
240
241static void
242busdma_init(void *dummy)
243{
244
245 maploads_total = counter_u64_alloc(M_WAITOK);
246 maploads_bounced = counter_u64_alloc(M_WAITOK);
247 maploads_coherent = counter_u64_alloc(M_WAITOK);
248 maploads_dmamem = counter_u64_alloc(M_WAITOK);
249 maploads_mbuf = counter_u64_alloc(M_WAITOK);
250 maploads_physmem = counter_u64_alloc(M_WAITOK);
214
215 /* Create a cache of buffers in standard (cacheable) memory. */
216 standard_allocator = busdma_bufalloc_create("buffer",
217 arm_dcache_align, /* minimum_alignment */
218 NULL, /* uma_alloc func */
219 NULL, /* uma_free func */
220 0); /* uma_zcreate_flags */
221

--- 228 unchanged lines hidden (view full) ---

450 /* Add pages to our bounce pool */
451 if (alloc_bounce_pages(newtag, pages) < pages)
452 error = ENOMEM;
453 }
454 /* Performed initial allocation */
455 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
456 } else
457 newtag->bounce_zone = NULL;
251
252 /* Create a cache of buffers in standard (cacheable) memory. */
253 standard_allocator = busdma_bufalloc_create("buffer",
254 arm_dcache_align, /* minimum_alignment */
255 NULL, /* uma_alloc func */
256 NULL, /* uma_free func */
257 0); /* uma_zcreate_flags */
258

--- 228 unchanged lines hidden (view full) ---

487 /* Add pages to our bounce pool */
488 if (alloc_bounce_pages(newtag, pages) < pages)
489 error = ENOMEM;
490 }
491 /* Performed initial allocation */
492 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
493 } else
494 newtag->bounce_zone = NULL;
458 if (error != 0)
495
496 if (error != 0) {
459 free(newtag, M_BUSDMA);
497 free(newtag, M_BUSDMA);
460 else
498 } else {
499 atomic_add_32(&tags_total, 1);
461 *dmat = newtag;
500 *dmat = newtag;
501 }
462 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
463 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
502 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
503 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
464
465 return (error);
466}
467
468int
469bus_dma_tag_destroy(bus_dma_tag_t dmat)
470{
504 return (error);
505}
506
507int
508bus_dma_tag_destroy(bus_dma_tag_t dmat)
509{
471#ifdef KTR
472 bus_dma_tag_t dmat_copy = dmat;
473#endif
510 bus_dma_tag_t dmat_copy;
511 int error;
474
512
513 error = 0;
514 dmat_copy = dmat;
515
475 if (dmat != NULL) {
476
516 if (dmat != NULL) {
517
477 if (dmat->map_count != 0)
478 return (EBUSY);
518 if (dmat->map_count != 0) {
519 error = EBUSY;
520 goto out;
521 }
479
480 while (dmat != NULL) {
481 bus_dma_tag_t parent;
482
483 parent = dmat->parent;
484 atomic_subtract_int(&dmat->ref_count, 1);
485 if (dmat->ref_count == 0) {
522
523 while (dmat != NULL) {
524 bus_dma_tag_t parent;
525
526 parent = dmat->parent;
527 atomic_subtract_int(&dmat->ref_count, 1);
528 if (dmat->ref_count == 0) {
529 atomic_subtract_32(&tags_total, 1);
486 free(dmat, M_BUSDMA);
487 /*
488 * Last reference count, so
489 * release our reference
490 * count on our parent.
491 */
492 dmat = parent;
493 } else
494 dmat = NULL;
495 }
496 }
530 free(dmat, M_BUSDMA);
531 /*
532 * Last reference count, so
533 * release our reference
534 * count on our parent.
535 */
536 dmat = parent;
537 } else
538 dmat = NULL;
539 }
540 }
497 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
541out:
542 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
543 return (error);
544}
498
545
546static int
547allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t map)
548{
549 int error;
550
551 /*
552 * Bouncing might be required if the driver asks for an active
553 * exclusion region, a data alignment that is stricter than 1, and/or
554 * an active address boundary.
555 */
556 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
557
558 /* Must bounce */
559 struct bounce_zone *bz;
560 int maxpages;
561
562 if (dmat->bounce_zone == NULL) {
563 if ((error = alloc_bounce_zone(dmat)) != 0) {
564 return (error);
565 }
566 }
567 bz = dmat->bounce_zone;
568
569 /* Initialize the new map */
570 STAILQ_INIT(&(map->bpages));
571
572 /*
573 * Attempt to add pages to our pool on a per-instance
574 * basis up to a sane limit.
575 */
576 maxpages = MAX_BPAGES;
577 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
578 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
579 int pages;
580
581 pages = MAX(atop(dmat->maxsize), 1);
582 pages = MIN(maxpages - bz->total_bpages, pages);
583 pages = MAX(pages, 1);
584 if (alloc_bounce_pages(dmat, pages) < pages)
585 return (ENOMEM);
586
587 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0)
588 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
589 }
590 bz->map_count++;
591 }
499 return (0);
500}
501
502static bus_dmamap_t
503allocate_map(bus_dma_tag_t dmat, int mflags)
504{
505 int mapsize, segsize;
506 bus_dmamap_t map;

--- 35 unchanged lines hidden (view full) ---

542
543 /*
544 * Bouncing might be required if the driver asks for an exclusion
545 * region, a data alignment that is stricter than 1, or DMA that begins
546 * or ends with a partial cacheline. Whether bouncing will actually
547 * happen can't be known until mapping time, but we need to pre-allocate
548 * resources now because we might not be allowed to at mapping time.
549 */
592 return (0);
593}
594
595static bus_dmamap_t
596allocate_map(bus_dma_tag_t dmat, int mflags)
597{
598 int mapsize, segsize;
599 bus_dmamap_t map;

--- 35 unchanged lines hidden (view full) ---

635
636 /*
637 * Bouncing might be required if the driver asks for an exclusion
638 * region, a data alignment that is stricter than 1, or DMA that begins
639 * or ends with a partial cacheline. Whether bouncing will actually
640 * happen can't be known until mapping time, but we need to pre-allocate
641 * resources now because we might not be allowed to at mapping time.
642 */
550 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
551
552 /* Must bounce */
553 struct bounce_zone *bz;
554 int maxpages;
555
556 if (dmat->bounce_zone == NULL) {
557 if ((error = alloc_bounce_zone(dmat)) != 0) {
558 free(map, M_BUSDMA);
559 *mapp = NULL;
560 return (error);
561 }
562 }
563 bz = dmat->bounce_zone;
564
565 /* Initialize the new map */
566 STAILQ_INIT(&((*mapp)->bpages));
567
568 /*
569 * Attempt to add pages to our pool on a per-instance
570 * basis up to a sane limit.
571 */
572 maxpages = MAX_BPAGES;
573 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
574 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
575 int pages;
576
577 pages = MAX(atop(dmat->maxsize), 1);
578 pages = MIN(maxpages - bz->total_bpages, pages);
579 pages = MAX(pages, 1);
580 if (alloc_bounce_pages(dmat, pages) < pages)
581 error = ENOMEM;
582
583 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
584 if (error == 0)
585 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
586 } else {
587 error = 0;
588 }
589 }
590 bz->map_count++;
643 error = allocate_bz_and_pages(dmat, map);
644 if (error != 0) {
645 free(map, M_BUSDMA);
646 *mapp = NULL;
647 return (error);
591 }
648 }
592 map->sync_count = 0;
593 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
594 __func__, dmat, dmat->flags, error);
649 if (map->flags & DMAMAP_COHERENT)
650 atomic_add_32(&maps_coherent, 1);
651 atomic_add_32(&maps_total, 1);
652 dmat->map_count++;
595
596 return (0);
597}
598
599/*
600 * Destroy a handle for mapping from kva/uva/physical
601 * address space into bus device space.
602 */
603int
604bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
605{
606
607 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
608 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
609 __func__, dmat, EBUSY);
610 return (EBUSY);
611 }
612 if (dmat->bounce_zone)
613 dmat->bounce_zone->map_count--;
653
654 return (0);
655}
656
657/*
658 * Destroy a handle for mapping from kva/uva/physical
659 * address space into bus device space.
660 */
661int
662bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
663{
664
665 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
666 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
667 __func__, dmat, EBUSY);
668 return (EBUSY);
669 }
670 if (dmat->bounce_zone)
671 dmat->bounce_zone->map_count--;
672 if (map->flags & DMAMAP_COHERENT)
673 atomic_subtract_32(&maps_coherent, 1);
674 atomic_subtract_32(&maps_total, 1);
614 free(map, M_BUSDMA);
615 dmat->map_count--;
616 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
617 return (0);
618}
619
620/*
621 * Allocate a piece of memory that can be efficiently mapped into bus device

--- 69 unchanged lines hidden (view full) ---

691 }
692 if (*vaddr == NULL) {
693 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
694 __func__, dmat, dmat->flags, ENOMEM);
695 free(map, M_BUSDMA);
696 *mapp = NULL;
697 return (ENOMEM);
698 }
675 free(map, M_BUSDMA);
676 dmat->map_count--;
677 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
678 return (0);
679}
680
681/*
682 * Allocate a piece of memory that can be efficiently mapped into bus device

--- 69 unchanged lines hidden (view full) ---

752 }
753 if (*vaddr == NULL) {
754 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
755 __func__, dmat, dmat->flags, ENOMEM);
756 free(map, M_BUSDMA);
757 *mapp = NULL;
758 return (ENOMEM);
759 }
760 if (map->flags & DMAMAP_COHERENT)
761 atomic_add_32(&maps_coherent, 1);
762 atomic_add_32(&maps_dmamem, 1);
763 atomic_add_32(&maps_total, 1);
699 dmat->map_count++;
700
701 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
702 __func__, dmat, dmat->flags, 0);
703 return (0);
704}
705
706/*

--- 15 unchanged lines hidden (view full) ---

722
723 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
724 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
725 uma_zfree(bufzone->umazone, vaddr);
726 else
727 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
728
729 dmat->map_count--;
764 dmat->map_count++;
765
766 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
767 __func__, dmat, dmat->flags, 0);
768 return (0);
769}
770
771/*

--- 15 unchanged lines hidden (view full) ---

787
788 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
789 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
790 uma_zfree(bufzone->umazone, vaddr);
791 else
792 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
793
794 dmat->map_count--;
795 if (map->flags & DMAMAP_COHERENT)
796 atomic_subtract_32(&maps_coherent, 1);
797 atomic_subtract_32(&maps_total, 1);
798 atomic_subtract_32(&maps_dmamem, 1);
730 free(map, M_BUSDMA);
731 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
732}
733
734static void
735_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
736 bus_size_t buflen, int flags)
737{

--- 139 unchanged lines hidden (view full) ---

877/*
878 * Utility function to load a physical buffer. segp contains
879 * the starting segment on entrace, and the ending segment on exit.
880 */
881int
882_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
883 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
884{
799 free(map, M_BUSDMA);
800 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
801}
802
803static void
804_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
805 bus_size_t buflen, int flags)
806{

--- 139 unchanged lines hidden (view full) ---

946/*
947 * Utility function to load a physical buffer. segp contains
948 * the starting segment on entrace, and the ending segment on exit.
949 */
950int
951_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
952 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
953{
885 struct sync_list *sl;
886 bus_size_t sgsize;
887 bus_addr_t curaddr;
888 bus_addr_t sl_end = 0;
954 bus_addr_t curaddr;
955 bus_addr_t sl_end = 0;
956 bus_size_t sgsize;
957 struct sync_list *sl;
889 int error;
890
891 if (segs == NULL)
892 segs = map->segments;
893
958 int error;
959
960 if (segs == NULL)
961 segs = map->segments;
962
963 counter_u64_add(maploads_total, 1);
964 counter_u64_add(maploads_physmem, 1);
965
894 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
895 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
896 if (map->pagesneeded != 0) {
966 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
967 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
968 if (map->pagesneeded != 0) {
969 counter_u64_add(maploads_bounced, 1);
897 error = _bus_dmamap_reserve_pages(dmat, map, flags);
898 if (error)
899 return (error);
900 }
901 }
902
903 sl = map->slist + map->sync_count - 1;
904

--- 62 unchanged lines hidden (view full) ---

967 bus_addr_t curaddr;
968 bus_addr_t sl_pend = 0;
969 struct sync_list *sl;
970 vm_offset_t kvaddr;
971 vm_offset_t vaddr = (vm_offset_t)buf;
972 vm_offset_t sl_vend = 0;
973 int error = 0;
974
970 error = _bus_dmamap_reserve_pages(dmat, map, flags);
971 if (error)
972 return (error);
973 }
974 }
975
976 sl = map->slist + map->sync_count - 1;
977

--- 62 unchanged lines hidden (view full) ---

1040 bus_addr_t curaddr;
1041 bus_addr_t sl_pend = 0;
1042 struct sync_list *sl;
1043 vm_offset_t kvaddr;
1044 vm_offset_t vaddr = (vm_offset_t)buf;
1045 vm_offset_t sl_vend = 0;
1046 int error = 0;
1047
1048 counter_u64_add(maploads_total, 1);
1049 if (map->flags & DMAMAP_COHERENT)
1050 counter_u64_add(maploads_coherent, 1);
1051 if (map->flags & DMAMAP_DMAMEM_ALLOC)
1052 counter_u64_add(maploads_dmamem, 1);
1053
975 if (segs == NULL)
976 segs = map->segments;
1054 if (segs == NULL)
1055 segs = map->segments;
977 if ((flags & BUS_DMA_LOAD_MBUF) != 0)
1056 if (flags & BUS_DMA_LOAD_MBUF) {
1057 counter_u64_add(maploads_mbuf, 1);
978 map->flags |= DMAMAP_CACHE_ALIGNED;
1058 map->flags |= DMAMAP_CACHE_ALIGNED;
1059 }
979
980 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
981 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
982 if (map->pagesneeded != 0) {
1060
1061 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
1062 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
1063 if (map->pagesneeded != 0) {
1064 counter_u64_add(maploads_bounced, 1);
983 error = _bus_dmamap_reserve_pages(dmat, map, flags);
984 if (error)
985 return (error);
986 }
987 }
988 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
989 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
990

--- 542 unchanged lines hidden --- 		1065 error = _bus_dmamap_reserve_pages(dmat, map, flags);
1066 if (error)
1067 return (error);
1068 }
1069 }
1070 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
1071 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
1072

--- 542 unchanged lines hidden ---