Deleted Added
sdiff udiff text old ( 286969 ) new ( 289759 )
full compact
busdma_machdep-v6.c (286969) busdma_machdep-v6.c (289759)
1/*-
2 * Copyright (c) 2012-2014 Ian Lepore
3 * Copyright (c) 2010 Mark Tinguely
4 * Copyright (c) 2004 Olivier Houchard
5 * Copyright (c) 2002 Peter Grehan
6 * Copyright (c) 1997, 1998 Justin T. Gibbs.
7 * All rights reserved.
8 *

--- 17 unchanged lines hidden (view full) ---

26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb
31 */
32
33#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2012-2014 Ian Lepore
3 * Copyright (c) 2010 Mark Tinguely
4 * Copyright (c) 2004 Olivier Houchard
5 * Copyright (c) 2002 Peter Grehan
6 * Copyright (c) 1997, 1998 Justin T. Gibbs.
7 * All rights reserved.
8 *

--- 17 unchanged lines hidden (view full) ---

26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep-v6.c 286969 2015-08-20 19:39:15Z ian $");
34__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep-v6.c 289759 2015-10-22 16:38:01Z jah $");
35
36#define _ARM32_BUS_DMA_PRIVATE
37#include <sys/param.h>
38#include <sys/kdb.h>
39#include <ddb/ddb.h>
40#include <ddb/db_output.h>
41#include <sys/systm.h>
42#include <sys/malloc.h>

--- 13 unchanged lines hidden (view full) ---

56#include <vm/vm.h>
57#include <vm/vm_page.h>
58#include <vm/vm_map.h>
59#include <vm/vm_extern.h>
60#include <vm/vm_kern.h>
61
62#include <machine/atomic.h>
63#include <machine/bus.h>
35
36#define _ARM32_BUS_DMA_PRIVATE
37#include <sys/param.h>
38#include <sys/kdb.h>
39#include <ddb/ddb.h>
40#include <ddb/db_output.h>
41#include <sys/systm.h>
42#include <sys/malloc.h>

--- 13 unchanged lines hidden (view full) ---

56#include <vm/vm.h>
57#include <vm/vm_page.h>
58#include <vm/vm_map.h>
59#include <vm/vm_extern.h>
60#include <vm/vm_kern.h>
61
62#include <machine/atomic.h>
63#include <machine/bus.h>
64#include <machine/cpufunc.h>
64#include <machine/cpu-v6.h>
65#include <machine/md_var.h>
66
67#define MAX_BPAGES 64
68#define MAX_DMA_SEGMENTS 4096
69#define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2
70#define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3
71#define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE)
72#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4

--- 26 unchanged lines hidden (view full) ---

99 struct arm32_dma_range *ranges;
100 int _nranges;
101};
102
103struct bounce_page {
104 vm_offset_t vaddr; /* kva of bounce buffer */
105 bus_addr_t busaddr; /* Physical address */
106 vm_offset_t datavaddr; /* kva of client data */
65#include <machine/md_var.h>
66
67#define MAX_BPAGES 64
68#define MAX_DMA_SEGMENTS 4096
69#define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2
70#define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3
71#define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE)
72#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4

--- 26 unchanged lines hidden (view full) ---

99 struct arm32_dma_range *ranges;
100 int _nranges;
101};
102
103struct bounce_page {
104 vm_offset_t vaddr; /* kva of bounce buffer */
105 bus_addr_t busaddr; /* Physical address */
106 vm_offset_t datavaddr; /* kva of client data */
107 bus_addr_t dataaddr; /* client physical address */
107 vm_page_t datapage; /* physical page of client data */
108 vm_offset_t dataoffs; /* page offset of client data */
108 bus_size_t datacount; /* client data count */
109 STAILQ_ENTRY(bounce_page) links;
110};
111
112struct sync_list {
113 vm_offset_t vaddr; /* kva of client data */
109 bus_size_t datacount; /* client data count */
110 STAILQ_ENTRY(bounce_page) links;
111};
112
113struct sync_list {
114 vm_offset_t vaddr; /* kva of client data */
114 bus_addr_t busaddr; /* client physical address */
115 vm_page_t pages; /* starting page of client data */
116 vm_offset_t dataoffs; /* page offset of client data */
115 bus_size_t datacount; /* client data count */
116};
117
118int busdma_swi_pending;
119
120struct bounce_zone {
121 STAILQ_ENTRY(bounce_zone) links;
122 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;

--- 53 unchanged lines hidden (view full) ---

176 "Total bounce pages");
177
178struct bus_dmamap {
179 struct bp_list bpages;
180 int pagesneeded;
181 int pagesreserved;
182 bus_dma_tag_t dmat;
183 struct memdesc mem;
117 bus_size_t datacount; /* client data count */
118};
119
120int busdma_swi_pending;
121
122struct bounce_zone {
123 STAILQ_ENTRY(bounce_zone) links;
124 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;

--- 53 unchanged lines hidden (view full) ---

178 "Total bounce pages");
179
180struct bus_dmamap {
181 struct bp_list bpages;
182 int pagesneeded;
183 int pagesreserved;
184 bus_dma_tag_t dmat;
185 struct memdesc mem;
184 pmap_t pmap;
185 bus_dmamap_callback_t *callback;
186 void *callback_arg;
187 int flags;
188#define DMAMAP_COHERENT (1 << 0)
189#define DMAMAP_DMAMEM_ALLOC (1 << 1)
190#define DMAMAP_MBUF (1 << 2)
191 STAILQ_ENTRY(bus_dmamap) links;
192 bus_dma_segment_t *segments;

--- 8 unchanged lines hidden (view full) ---

201static int alloc_bounce_zone(bus_dma_tag_t dmat);
202static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
203static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
204 int commit);
205static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
206 vm_offset_t vaddr, bus_addr_t addr,
207 bus_size_t size);
208static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
186 bus_dmamap_callback_t *callback;
187 void *callback_arg;
188 int flags;
189#define DMAMAP_COHERENT (1 << 0)
190#define DMAMAP_DMAMEM_ALLOC (1 << 1)
191#define DMAMAP_MBUF (1 << 2)
192 STAILQ_ENTRY(bus_dmamap) links;
193 bus_dma_segment_t *segments;

--- 8 unchanged lines hidden (view full) ---

202static int alloc_bounce_zone(bus_dma_tag_t dmat);
203static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
204static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
205 int commit);
206static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
207 vm_offset_t vaddr, bus_addr_t addr,
208 bus_size_t size);
209static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
209static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
210 void *buf, bus_size_t buflen, int flags);
210static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap,
211 bus_dmamap_t map, void *buf, bus_size_t buflen, int flags);
211static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
212 vm_paddr_t buf, bus_size_t buflen, int flags);
213static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
214 int flags);
212static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
213 vm_paddr_t buf, bus_size_t buflen, int flags);
214static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
215 int flags);
216static void dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size);
217static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op);
215
216static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
217static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
218static void
219busdma_init(void *dummy)
220{
221 int uma_flags;
222

--- 668 unchanged lines hidden (view full) ---

891 /*
892 * Count the number of bounce pages
893 * needed in order to complete this transfer
894 */
895 curaddr = buf;
896 while (buflen != 0) {
897 sgsize = MIN(buflen, dmat->maxsegsz);
898 if (must_bounce(dmat, map, curaddr, sgsize) != 0) {
218
219static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
220static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
221static void
222busdma_init(void *dummy)
223{
224 int uma_flags;
225

--- 668 unchanged lines hidden (view full) ---

894 /*
895 * Count the number of bounce pages
896 * needed in order to complete this transfer
897 */
898 curaddr = buf;
899 while (buflen != 0) {
900 sgsize = MIN(buflen, dmat->maxsegsz);
901 if (must_bounce(dmat, map, curaddr, sgsize) != 0) {
899 sgsize = MIN(sgsize, PAGE_SIZE);
902 sgsize = MIN(sgsize,
903 PAGE_SIZE - (curaddr & PAGE_MASK));
900 map->pagesneeded++;
901 }
902 curaddr += sgsize;
903 buflen -= sgsize;
904 }
905 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded);
906 }
907}
908
909static void
904 map->pagesneeded++;
905 }
906 curaddr += sgsize;
907 buflen -= sgsize;
908 }
909 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded);
910 }
911}
912
913static void
910_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
914_bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map,
911 void *buf, bus_size_t buflen, int flags)
912{
913 vm_offset_t vaddr;
914 vm_offset_t vendaddr;
915 bus_addr_t paddr;
916
917 if (map->pagesneeded == 0) {
918 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d"
919 " map= %p, pagesneeded= %d",
920 dmat->lowaddr, dmat->boundary, dmat->alignment,
921 map, map->pagesneeded);
922 /*
923 * Count the number of bounce pages
924 * needed in order to complete this transfer
925 */
926 vaddr = (vm_offset_t)buf;
927 vendaddr = (vm_offset_t)buf + buflen;
928
929 while (vaddr < vendaddr) {
915 void *buf, bus_size_t buflen, int flags)
916{
917 vm_offset_t vaddr;
918 vm_offset_t vendaddr;
919 bus_addr_t paddr;
920
921 if (map->pagesneeded == 0) {
922 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d"
923 " map= %p, pagesneeded= %d",
924 dmat->lowaddr, dmat->boundary, dmat->alignment,
925 map, map->pagesneeded);
926 /*
927 * Count the number of bounce pages
928 * needed in order to complete this transfer
929 */
930 vaddr = (vm_offset_t)buf;
931 vendaddr = (vm_offset_t)buf + buflen;
932
933 while (vaddr < vendaddr) {
930 if (__predict_true(map->pmap == kernel_pmap))
934 if (__predict_true(pmap == kernel_pmap))
931 paddr = pmap_kextract(vaddr);
932 else
935 paddr = pmap_kextract(vaddr);
936 else
933 paddr = pmap_extract(map->pmap, vaddr);
937 paddr = pmap_extract(pmap, vaddr);
934 if (must_bounce(dmat, map, paddr,
935 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr &
936 PAGE_MASK)))) != 0) {
937 map->pagesneeded++;
938 }
939 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
940
941 }

--- 96 unchanged lines hidden (view full) ---

1038_bus_dmamap_load_phys(bus_dma_tag_t dmat,
1039 bus_dmamap_t map,
1040 vm_paddr_t buf, bus_size_t buflen,
1041 int flags,
1042 bus_dma_segment_t *segs,
1043 int *segp)
1044{
1045 bus_addr_t curaddr;
938 if (must_bounce(dmat, map, paddr,
939 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr &
940 PAGE_MASK)))) != 0) {
941 map->pagesneeded++;
942 }
943 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
944
945 }

--- 96 unchanged lines hidden (view full) ---

1042_bus_dmamap_load_phys(bus_dma_tag_t dmat,
1043 bus_dmamap_t map,
1044 vm_paddr_t buf, bus_size_t buflen,
1045 int flags,
1046 bus_dma_segment_t *segs,
1047 int *segp)
1048{
1049 bus_addr_t curaddr;
1050 bus_addr_t sl_end = 0;
1046 bus_size_t sgsize;
1051 bus_size_t sgsize;
1052 struct sync_list *sl;
1047 int error;
1048
1049 if (segs == NULL)
1050 segs = map->segments;
1051
1052 counter_u64_add(maploads_total, 1);
1053 counter_u64_add(maploads_physmem, 1);
1054
1053 int error;
1054
1055 if (segs == NULL)
1056 segs = map->segments;
1057
1058 counter_u64_add(maploads_total, 1);
1059 counter_u64_add(maploads_physmem, 1);
1060
1055 if (might_bounce(dmat, map, buflen, buflen)) {
1061 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
1056 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
1057 if (map->pagesneeded != 0) {
1058 counter_u64_add(maploads_bounced, 1);
1059 error = _bus_dmamap_reserve_pages(dmat, map, flags);
1060 if (error)
1061 return (error);
1062 }
1063 }
1064
1062 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
1063 if (map->pagesneeded != 0) {
1064 counter_u64_add(maploads_bounced, 1);
1065 error = _bus_dmamap_reserve_pages(dmat, map, flags);
1066 if (error)
1067 return (error);
1068 }
1069 }
1070
1071 sl = map->slist + map->sync_count - 1;
1072
1065 while (buflen > 0) {
1066 curaddr = buf;
1067 sgsize = MIN(buflen, dmat->maxsegsz);
1068 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
1069 sgsize)) {
1073 while (buflen > 0) {
1074 curaddr = buf;
1075 sgsize = MIN(buflen, dmat->maxsegsz);
1076 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
1077 sgsize)) {
1070 sgsize = MIN(sgsize, PAGE_SIZE);
1078 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
1071 curaddr = add_bounce_page(dmat, map, 0, curaddr,
1072 sgsize);
1079 curaddr = add_bounce_page(dmat, map, 0, curaddr,
1080 sgsize);
1081 } else {
1082 if (map->sync_count > 0)
1083 sl_end = VM_PAGE_TO_PHYS(sl->pages) +
1084 sl->dataoffs + sl->datacount;
1085
1086 if (map->sync_count == 0 || curaddr != sl_end) {
1087 if (++map->sync_count > dmat->nsegments)
1088 break;
1089 sl++;
1090 sl->vaddr = 0;
1091 sl->datacount = sgsize;
1092 /*
1093 * PHYS_TO_VM_PAGE() will truncate
1094 * unaligned addresses.
1095 */
1096 sl->pages = PHYS_TO_VM_PAGE(curaddr);
1097 sl->dataoffs = curaddr & PAGE_MASK;
1098 } else
1099 sl->datacount += sgsize;
1073 }
1074 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1075 segp);
1076 if (sgsize == 0)
1077 break;
1078 buf += sgsize;
1079 buflen -= sgsize;
1080 }

--- 28 unchanged lines hidden (view full) ---

1109 void *buf, bus_size_t buflen,
1110 pmap_t pmap,
1111 int flags,
1112 bus_dma_segment_t *segs,
1113 int *segp)
1114{
1115 bus_size_t sgsize;
1116 bus_addr_t curaddr;
1100 }
1101 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1102 segp);
1103 if (sgsize == 0)
1104 break;
1105 buf += sgsize;
1106 buflen -= sgsize;
1107 }

--- 28 unchanged lines hidden (view full) ---

1136 void *buf, bus_size_t buflen,
1137 pmap_t pmap,
1138 int flags,
1139 bus_dma_segment_t *segs,
1140 int *segp)
1141{
1142 bus_size_t sgsize;
1143 bus_addr_t curaddr;
1117 vm_offset_t vaddr;
1144 bus_addr_t sl_pend = 0;
1145 vm_offset_t kvaddr, vaddr, sl_vend = 0;
1118 struct sync_list *sl;
1119 int error;
1120
1121 counter_u64_add(maploads_total, 1);
1122 if (map->flags & DMAMAP_COHERENT)
1123 counter_u64_add(maploads_coherent, 1);
1124 if (map->flags & DMAMAP_DMAMEM_ALLOC)
1125 counter_u64_add(maploads_dmamem, 1);
1126
1127 if (segs == NULL)
1128 segs = map->segments;
1129
1130 if (flags & BUS_DMA_LOAD_MBUF) {
1131 counter_u64_add(maploads_mbuf, 1);
1132 map->flags |= DMAMAP_MBUF;
1133 }
1134
1146 struct sync_list *sl;
1147 int error;
1148
1149 counter_u64_add(maploads_total, 1);
1150 if (map->flags & DMAMAP_COHERENT)
1151 counter_u64_add(maploads_coherent, 1);
1152 if (map->flags & DMAMAP_DMAMEM_ALLOC)
1153 counter_u64_add(maploads_dmamem, 1);
1154
1155 if (segs == NULL)
1156 segs = map->segments;
1157
1158 if (flags & BUS_DMA_LOAD_MBUF) {
1159 counter_u64_add(maploads_mbuf, 1);
1160 map->flags |= DMAMAP_MBUF;
1161 }
1162
1135 map->pmap = pmap;
1136
1137 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
1163 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
1138 _bus_dmamap_count_pages(dmat, map, buf, buflen, flags);
1164 _bus_dmamap_count_pages(dmat, pmap, map, buf, buflen, flags);
1139 if (map->pagesneeded != 0) {
1140 counter_u64_add(maploads_bounced, 1);
1141 error = _bus_dmamap_reserve_pages(dmat, map, flags);
1142 if (error)
1143 return (error);
1144 }
1145 }
1146
1165 if (map->pagesneeded != 0) {
1166 counter_u64_add(maploads_bounced, 1);
1167 error = _bus_dmamap_reserve_pages(dmat, map, flags);
1168 if (error)
1169 return (error);
1170 }
1171 }
1172
1147 sl = NULL;
1173 sl = map->slist + map->sync_count - 1;
1148 vaddr = (vm_offset_t)buf;
1149
1150 while (buflen > 0) {
1151 /*
1152 * Get the physical address for this segment.
1153 */
1174 vaddr = (vm_offset_t)buf;
1175
1176 while (buflen > 0) {
1177 /*
1178 * Get the physical address for this segment.
1179 */
1154 if (__predict_true(map->pmap == kernel_pmap))
1180 if (__predict_true(pmap == kernel_pmap)) {
1155 curaddr = pmap_kextract(vaddr);
1181 curaddr = pmap_kextract(vaddr);
1156 else
1157 curaddr = pmap_extract(map->pmap, vaddr);
1182 kvaddr = vaddr;
1183 } else {
1184 curaddr = pmap_extract(pmap, vaddr);
1185 kvaddr = 0;
1186 }
1158
1159 /*
1160 * Compute the segment size, and adjust counts.
1161 */
1187
1188 /*
1189 * Compute the segment size, and adjust counts.
1190 */
1162 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
1191 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
1163 if (sgsize > dmat->maxsegsz)
1164 sgsize = dmat->maxsegsz;
1165 if (buflen < sgsize)
1166 sgsize = buflen;
1167
1168 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
1169 sgsize)) {
1192 if (sgsize > dmat->maxsegsz)
1193 sgsize = dmat->maxsegsz;
1194 if (buflen < sgsize)
1195 sgsize = buflen;
1196
1197 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
1198 sgsize)) {
1170 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
1199 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
1171 sgsize);
1172 } else {
1200 sgsize);
1201 } else {
1173 sl = &map->slist[map->sync_count - 1];
1202 if (map->sync_count > 0) {
1203 sl_pend = VM_PAGE_TO_PHYS(sl->pages) +
1204 sl->dataoffs + sl->datacount;
1205 sl_vend = sl->vaddr + sl->datacount;
1206 }
1207
1174 if (map->sync_count == 0 ||
1208 if (map->sync_count == 0 ||
1175#ifdef ARM_L2_PIPT
1176 curaddr != sl->busaddr + sl->datacount ||
1177#endif
1178 vaddr != sl->vaddr + sl->datacount) {
1209 (kvaddr != 0 && kvaddr != sl_vend) ||
1210 (curaddr != sl_pend)) {
1211
1179 if (++map->sync_count > dmat->nsegments)
1180 goto cleanup;
1181 sl++;
1212 if (++map->sync_count > dmat->nsegments)
1213 goto cleanup;
1214 sl++;
1182 sl->vaddr = vaddr;
1215 sl->vaddr = kvaddr;
1183 sl->datacount = sgsize;
1216 sl->datacount = sgsize;
1184 sl->busaddr = curaddr;
1217 /*
1218 * PHYS_TO_VM_PAGE() will truncate
1219 * unaligned addresses.
1220 */
1221 sl->pages = PHYS_TO_VM_PAGE(curaddr);
1222 sl->dataoffs = curaddr & PAGE_MASK;
1185 } else
1186 sl->datacount += sgsize;
1187 }
1188 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1189 segp);
1190 if (sgsize == 0)
1191 break;
1192 vaddr += sgsize;

--- 54 unchanged lines hidden (view full) ---

1247 bz->reserved_bpages -= map->pagesreserved;
1248 map->pagesreserved = 0;
1249 map->pagesneeded = 0;
1250 }
1251 map->sync_count = 0;
1252 map->flags &= ~DMAMAP_MBUF;
1253}
1254
1223 } else
1224 sl->datacount += sgsize;
1225 }
1226 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1227 segp);
1228 if (sgsize == 0)
1229 break;
1230 vaddr += sgsize;

--- 54 unchanged lines hidden (view full) ---

1285 bz->reserved_bpages -= map->pagesreserved;
1286 map->pagesreserved = 0;
1287 map->pagesneeded = 0;
1288 }
1289 map->sync_count = 0;
1290 map->flags &= ~DMAMAP_MBUF;
1291}
1292
1255#ifdef notyetbounceuser
1256/* If busdma uses user pages, then the interrupt handler could
1257 * be use the kernel vm mapping. Both bounce pages and sync list
1258 * do not cross page boundaries.
1259 * Below is a rough sequence that a person would do to fix the
1260 * user page reference in the kernel vmspace. This would be
1261 * done in the dma post routine.
1262 */
1263void
1264_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len,
1265 pmap_t pmap, int op)
1293static void
1294dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size)
1266{
1295{
1267 bus_size_t sgsize;
1268 bus_addr_t curaddr;
1269 vm_offset_t va;
1270
1271 /*
1296 /*
1272 * each synclist entry is contained within a single page.
1273 * this would be needed if BUS_DMASYNC_POSTxxxx was implemented
1297 * Write back any partial cachelines immediately before and
1298 * after the DMA region. We don't need to round the address
1299 * down to the nearest cacheline or specify the exact size,
1300 * as dcache_wb_poc() will do the rounding for us and works
1301 * at cacheline granularity.
1274 */
1302 */
1275 curaddr = pmap_extract(pmap, buf);
1276 va = pmap_dma_map(curaddr);
1277 switch (op) {
1278 case SYNC_USER_INV:
1279 cpu_dcache_wb_range(va, sgsize);
1280 break;
1303 if (va & cpuinfo.dcache_line_mask)
1304 dcache_wb_poc(va, pa, 1);
1305 if ((va + size) & cpuinfo.dcache_line_mask)
1306 dcache_wb_poc(va + size, pa + size, 1);
1281
1307
1282 case SYNC_USER_COPYTO:
1283 bcopy((void *)va, (void *)bounce, sgsize);
1284 break;
1308 dcache_dma_preread(va, pa, size);
1309}
1285
1310
1286 case SYNC_USER_COPYFROM:
1287 bcopy((void *) bounce, (void *)va, sgsize);
1288 break;
1311static void
1312dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op)
1313{
1314 uint32_t len, offset;
1315 vm_page_t m;
1316 vm_paddr_t pa;
1317 vm_offset_t va, tempva;
1318 bus_size_t size;
1289
1319
1290 default:
1291 break;
1292 }
1320 offset = sl->dataoffs;
1321 m = sl->pages;
1322 size = sl->datacount;
1323 pa = VM_PAGE_TO_PHYS(m) | offset;
1293
1324
1294 pmap_dma_unmap(va);
1295}
1296#endif
1325 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) {
1326 tempva = 0;
1327 if (sl->vaddr == 0) {
1328 len = min(PAGE_SIZE - offset, size);
1329 tempva = pmap_quick_enter_page(m);
1330 va = tempva | offset;
1331 } else {
1332 len = sl->datacount;
1333 va = sl->vaddr;
1334 }
1335 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset),
1336 ("unexpected vm_page_t phys: 0x%08x != 0x%08x",
1337 VM_PAGE_TO_PHYS(m) | offset, pa));
1297
1338
1298#ifdef ARM_L2_PIPT
1299#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size)
1300#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size)
1301#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size)
1302#else
1303#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size)
1304#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size)
1305#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size)
1306#endif
1339 switch (op) {
1340 case BUS_DMASYNC_PREWRITE:
1341 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
1342 dcache_wb_poc(va, pa, len);
1343 break;
1344 case BUS_DMASYNC_PREREAD:
1345 /*
1346 * An mbuf may start in the middle of a cacheline. There
1347 * will be no cpu writes to the beginning of that line
1348 * (which contains the mbuf header) while dma is in
1349 * progress. Handle that case by doing a writeback of
1350 * just the first cacheline before invalidating the
1351 * overall buffer. Any mbuf in a chain may have this
1352 * misalignment. Buffers which are not mbufs bounce if
1353 * they are not aligned to a cacheline.
1354 */
1355 dma_preread_safe(va, pa, len);
1356 break;
1357 case BUS_DMASYNC_POSTREAD:
1358 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
1359 dcache_inv_poc(va, pa, len);
1360 break;
1361 default:
1362 panic("unsupported combination of sync operations: "
1363 "0x%08x\n", op);
1364 }
1307
1365
1366 if (tempva != 0)
1367 pmap_quick_remove_page(tempva);
1368 }
1369}
1370
1308void
1309_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1310{
1311 struct bounce_page *bpage;
1312 struct sync_list *sl, *end;
1371void
1372_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1373{
1374 struct bounce_page *bpage;
1375 struct sync_list *sl, *end;
1376 vm_offset_t datavaddr, tempvaddr;
1377
1378 if (op == BUS_DMASYNC_POSTWRITE)
1379 return;
1380
1313 /*
1314 * If the buffer was from user space, it is possible that this is not
1315 * the same vm map, especially on a POST operation. It's not clear that
1316 * dma on userland buffers can work at all right now. To be safe, until
1317 * we're able to test direct userland dma, panic on a map mismatch.
1318 */
1319 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1381 /*
1382 * If the buffer was from user space, it is possible that this is not
1383 * the same vm map, especially on a POST operation. It's not clear that
1384 * dma on userland buffers can work at all right now. To be safe, until
1385 * we're able to test direct userland dma, panic on a map mismatch.
1386 */
1387 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1320 if (!pmap_dmap_iscurrent(map->pmap))
1321 panic("_bus_dmamap_sync: wrong user map for bounce sync.");
1322
1323 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
1324 "performing bounce", __func__, dmat, dmat->flags, op);
1325
1326 /*
1327 * For PREWRITE do a writeback. Clean the caches from the
1328 * innermost to the outermost levels.
1329 */
1330 if (op & BUS_DMASYNC_PREWRITE) {
1331 while (bpage != NULL) {
1388
1389 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
1390 "performing bounce", __func__, dmat, dmat->flags, op);
1391
1392 /*
1393 * For PREWRITE do a writeback. Clean the caches from the
1394 * innermost to the outermost levels.
1395 */
1396 if (op & BUS_DMASYNC_PREWRITE) {
1397 while (bpage != NULL) {
1332 if (bpage->datavaddr != 0)
1333 bcopy((void *)bpage->datavaddr,
1334 (void *)bpage->vaddr,
1335 bpage->datacount);
1336 else
1337 physcopyout(bpage->dataaddr,
1338 (void *)bpage->vaddr,
1339 bpage->datacount);
1340 cpu_dcache_wb_range((vm_offset_t)bpage->vaddr,
1398 tempvaddr = 0;
1399 datavaddr = bpage->datavaddr;
1400 if (datavaddr == 0) {
1401 tempvaddr = pmap_quick_enter_page(
1402 bpage->datapage);
1403 datavaddr = tempvaddr | bpage->dataoffs;
1404 }
1405 bcopy((void *)datavaddr, (void *)bpage->vaddr,
1341 bpage->datacount);
1406 bpage->datacount);
1342 l2cache_wb_range((vm_offset_t)bpage->vaddr,
1343 (vm_offset_t)bpage->busaddr,
1407 if (tempvaddr != 0)
1408 pmap_quick_remove_page(tempvaddr);
1409 dcache_wb_poc(bpage->vaddr, bpage->busaddr,
1344 bpage->datacount);
1345 bpage = STAILQ_NEXT(bpage, links);
1346 }
1347 dmat->bounce_zone->total_bounced++;
1348 }
1349
1350 /*
1351 * Do an invalidate for PREREAD unless a writeback was already

--- 4 unchanged lines hidden (view full) ---

1356 * lines and the POSTREAD invalidate handles the rest. The
1357 * invalidate is done from the innermost to outermost level. If
1358 * L2 were done first, a dirty cacheline could be automatically
1359 * evicted from L1 before we invalidated it, re-dirtying the L2.
1360 */
1361 if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
1362 bpage = STAILQ_FIRST(&map->bpages);
1363 while (bpage != NULL) {
1410 bpage->datacount);
1411 bpage = STAILQ_NEXT(bpage, links);
1412 }
1413 dmat->bounce_zone->total_bounced++;
1414 }
1415
1416 /*
1417 * Do an invalidate for PREREAD unless a writeback was already

--- 4 unchanged lines hidden (view full) ---

1422 * lines and the POSTREAD invalidate handles the rest. The
1423 * invalidate is done from the innermost to outermost level. If
1424 * L2 were done first, a dirty cacheline could be automatically
1425 * evicted from L1 before we invalidated it, re-dirtying the L2.
1426 */
1427 if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
1428 bpage = STAILQ_FIRST(&map->bpages);
1429 while (bpage != NULL) {
1364 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr,
1430 dcache_dma_preread(bpage->vaddr, bpage->busaddr,
1365 bpage->datacount);
1431 bpage->datacount);
1366 l2cache_inv_range((vm_offset_t)bpage->vaddr,
1367 (vm_offset_t)bpage->busaddr,
1368 bpage->datacount);
1369 bpage = STAILQ_NEXT(bpage, links);
1370 }
1371 }
1372
1373 /*
1374 * Re-invalidate the caches on a POSTREAD, even though they were
1375 * already invalidated at PREREAD time. Aggressive prefetching
1376 * due to accesses to other data near the dma buffer could have
1377 * brought buffer data into the caches which is now stale. The
1378 * caches are invalidated from the outermost to innermost; the
1379 * prefetches could be happening right now, and if L1 were
1380 * invalidated first, stale L2 data could be prefetched into L1.
1381 */
1382 if (op & BUS_DMASYNC_POSTREAD) {
1383 while (bpage != NULL) {
1432 bpage = STAILQ_NEXT(bpage, links);
1433 }
1434 }
1435
1436 /*
1437 * Re-invalidate the caches on a POSTREAD, even though they were
1438 * already invalidated at PREREAD time. Aggressive prefetching
1439 * due to accesses to other data near the dma buffer could have
1440 * brought buffer data into the caches which is now stale. The
1441 * caches are invalidated from the outermost to innermost; the
1442 * prefetches could be happening right now, and if L1 were
1443 * invalidated first, stale L2 data could be prefetched into L1.
1444 */
1445 if (op & BUS_DMASYNC_POSTREAD) {
1446 while (bpage != NULL) {
1384 l2cache_inv_range((vm_offset_t)bpage->vaddr,
1385 (vm_offset_t)bpage->busaddr,
1447 dcache_inv_poc(bpage->vaddr, bpage->busaddr,
1386 bpage->datacount);
1448 bpage->datacount);
1387 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr,
1449 tempvaddr = 0;
1450 datavaddr = bpage->datavaddr;
1451 if (datavaddr == 0) {
1452 tempvaddr = pmap_quick_enter_page(
1453 bpage->datapage);
1454 datavaddr = tempvaddr | bpage->dataoffs;
1455 }
1456 bcopy((void *)bpage->vaddr, (void *)datavaddr,
1388 bpage->datacount);
1457 bpage->datacount);
1389 if (bpage->datavaddr != 0)
1390 bcopy((void *)bpage->vaddr,
1391 (void *)bpage->datavaddr,
1392 bpage->datacount);
1393 else
1394 physcopyin((void *)bpage->vaddr,
1395 bpage->dataaddr,
1396 bpage->datacount);
1458 if (tempvaddr != 0)
1459 pmap_quick_remove_page(tempvaddr);
1397 bpage = STAILQ_NEXT(bpage, links);
1398 }
1399 dmat->bounce_zone->total_bounced++;
1400 }
1401 }
1402
1403 /*
1404 * For COHERENT memory no cache maintenance is necessary, but ensure all

--- 14 unchanged lines hidden (view full) ---

1419 /*
1420 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All
1421 * the comments about the sequences for flushing cache levels in the
1422 * bounce buffer code above apply here as well. In particular, the fact
1423 * that the sequence is inner-to-outer for PREREAD invalidation and
1424 * outer-to-inner for POSTREAD invalidation is not a mistake.
1425 */
1426 if (map->sync_count != 0) {
1460 bpage = STAILQ_NEXT(bpage, links);
1461 }
1462 dmat->bounce_zone->total_bounced++;
1463 }
1464 }
1465
1466 /*
1467 * For COHERENT memory no cache maintenance is necessary, but ensure all

--- 14 unchanged lines hidden (view full) ---

1482 /*
1483 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All
1484 * the comments about the sequences for flushing cache levels in the
1485 * bounce buffer code above apply here as well. In particular, the fact
1486 * that the sequence is inner-to-outer for PREREAD invalidation and
1487 * outer-to-inner for POSTREAD invalidation is not a mistake.
1488 */
1489 if (map->sync_count != 0) {
1427 if (!pmap_dmap_iscurrent(map->pmap))
1428 panic("_bus_dmamap_sync: wrong user map for sync.");
1429
1430 sl = &map->slist[0];
1431 end = &map->slist[map->sync_count];
1432 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
1433 "performing sync", __func__, dmat, dmat->flags, op);
1434
1490 sl = &map->slist[0];
1491 end = &map->slist[map->sync_count];
1492 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
1493 "performing sync", __func__, dmat, dmat->flags, op);
1494
1435 switch (op) {
1436 case BUS_DMASYNC_PREWRITE:
1437 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
1438 while (sl != end) {
1439 cpu_dcache_wb_range(sl->vaddr, sl->datacount);
1440 l2cache_wb_range(sl->vaddr, sl->busaddr,
1441 sl->datacount);
1442 sl++;
1443 }
1444 break;
1445
1446 case BUS_DMASYNC_PREREAD:
1447 /*
1448 * An mbuf may start in the middle of a cacheline. There
1449 * will be no cpu writes to the beginning of that line
1450 * (which contains the mbuf header) while dma is in
1451 * progress. Handle that case by doing a writeback of
1452 * just the first cacheline before invalidating the
1453 * overall buffer. Any mbuf in a chain may have this
1454 * misalignment. Buffers which are not mbufs bounce if
1455 * they are not aligned to a cacheline.
1456 */
1457 while (sl != end) {
1458 if (sl->vaddr & arm_dcache_align_mask) {
1459 KASSERT(map->flags & DMAMAP_MBUF,
1460 ("unaligned buffer is not an mbuf"));
1461 cpu_dcache_wb_range(sl->vaddr, 1);
1462 l2cache_wb_range(sl->vaddr,
1463 sl->busaddr, 1);
1464 }
1465 cpu_dcache_inv_range(sl->vaddr, sl->datacount);
1466 l2cache_inv_range(sl->vaddr, sl->busaddr,
1467 sl->datacount);
1468 sl++;
1469 }
1470 break;
1471
1472 case BUS_DMASYNC_POSTWRITE:
1473 break;
1474
1475 case BUS_DMASYNC_POSTREAD:
1476 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
1477 while (sl != end) {
1478 l2cache_inv_range(sl->vaddr, sl->busaddr,
1479 sl->datacount);
1480 cpu_dcache_inv_range(sl->vaddr, sl->datacount);
1481 sl++;
1482 }
1483 break;
1484
1485 default:
1486 panic("unsupported combination of sync operations: 0x%08x\n", op);
1487 break;
1488 }
1495 for ( ; sl != end; ++sl)
1496 dma_dcache_sync(sl, op);
1489 }
1490}
1491
1492static void
1493init_bounce_pages(void *dummy __unused)
1494{
1495
1496 total_bpages = 0;

--- 177 unchanged lines hidden (view full) ---

1674 mtx_unlock(&bounce_lock);
1675
1676 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1677 /* Page offset needs to be preserved. */
1678 bpage->vaddr |= addr & PAGE_MASK;
1679 bpage->busaddr |= addr & PAGE_MASK;
1680 }
1681 bpage->datavaddr = vaddr;
1497 }
1498}
1499
1500static void
1501init_bounce_pages(void *dummy __unused)
1502{
1503
1504 total_bpages = 0;

--- 177 unchanged lines hidden (view full) ---

1682 mtx_unlock(&bounce_lock);
1683
1684 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1685 /* Page offset needs to be preserved. */
1686 bpage->vaddr |= addr & PAGE_MASK;
1687 bpage->busaddr |= addr & PAGE_MASK;
1688 }
1689 bpage->datavaddr = vaddr;
1682 bpage->dataaddr = addr;
1690 /* PHYS_TO_VM_PAGE() will truncate unaligned addresses. */
1691 bpage->datapage = PHYS_TO_VM_PAGE(addr);
1692 bpage->dataoffs = addr & PAGE_MASK;
1683 bpage->datacount = size;
1684 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1685 return (bpage->busaddr);
1686}
1687
1688static void
1689free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1690{

--- 52 unchanged lines hidden --- 		1693 bpage->datacount = size;
1694 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1695 return (bpage->busaddr);
1696}
1697
1698static void
1699free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1700{

--- 52 unchanged lines hidden ---