Deleted Added
sdiff udiff text old ( 257228 ) new ( 257230 )
full compact
busdma_bounce.c (257228) busdma_bounce.c (257230)
1/*-
2 * Copyright (c) 1997, 1998 Justin T. Gibbs.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1997, 1998 Justin T. Gibbs.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/x86/x86/busdma_machdep.c 257228 2013-10-27 21:39:16Z kib $");
28__FBSDID("$FreeBSD: head/sys/x86/x86/busdma_bounce.c 257230 2013-10-27 22:05:10Z kib $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/bus.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>
36#include <sys/ktr.h>

--- 9 unchanged lines hidden (view full) ---

46#include <vm/vm_kern.h>
47#include <vm/vm_page.h>
48#include <vm/vm_map.h>
49
50#include <machine/atomic.h>
51#include <machine/bus.h>
52#include <machine/md_var.h>
53#include <machine/specialreg.h>
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/bus.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>
36#include <sys/ktr.h>

--- 9 unchanged lines hidden (view full) ---

46#include <vm/vm_kern.h>
47#include <vm/vm_page.h>
48#include <vm/vm_map.h>
49
50#include <machine/atomic.h>
51#include <machine/bus.h>
52#include <machine/md_var.h>
53#include <machine/specialreg.h>
54#include <x86/include/busdma_impl.h>
54
55#ifdef __i386__
56#define MAX_BPAGES 512
57#else
58#define MAX_BPAGES 8192
59#endif
60#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
61#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
62
63struct bounce_zone;
64
65struct bus_dma_tag {
55
56#ifdef __i386__
57#define MAX_BPAGES 512
58#else
59#define MAX_BPAGES 8192
60#endif
61#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
62#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
63
64struct bounce_zone;
65
66struct bus_dma_tag {
66 bus_dma_tag_t parent;
67 bus_size_t alignment;
68 bus_addr_t boundary;
69 bus_addr_t lowaddr;
70 bus_addr_t highaddr;
71 bus_dma_filter_t *filter;
72 void *filterarg;
73 bus_size_t maxsize;
74 u_int nsegments;
75 bus_size_t maxsegsz;
76 int flags;
77 int ref_count;
67 struct bus_dma_tag_common common;
78 int map_count;
68 int map_count;
79 bus_dma_lock_t *lockfunc;
80 void *lockfuncarg;
81 bus_dma_segment_t *segments;
82 struct bounce_zone *bounce_zone;
83};
84
85struct bounce_page {
86 vm_offset_t vaddr; /* kva of bounce buffer */
87 bus_addr_t busaddr; /* Physical address */
88 vm_offset_t datavaddr; /* kva of client data */

--- 66 unchanged lines hidden (view full) ---

155 int flags);
156
157#ifdef XEN
158#undef pmap_kextract
159#define pmap_kextract pmap_kextract_ma
160#endif
161
162/*
69 bus_dma_segment_t *segments;
70 struct bounce_zone *bounce_zone;
71};
72
73struct bounce_page {
74 vm_offset_t vaddr; /* kva of bounce buffer */
75 bus_addr_t busaddr; /* Physical address */
76 vm_offset_t datavaddr; /* kva of client data */

--- 66 unchanged lines hidden (view full) ---

143 int flags);
144
145#ifdef XEN
146#undef pmap_kextract
147#define pmap_kextract pmap_kextract_ma
148#endif
149
150/*
163 * Return true if a match is made.
164 *
165 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
166 *
167 * If paddr is within the bounds of the dma tag then call the filter callback
168 * to check for a match, if there is no filter callback then assume a match.
169 */
170int
171run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
172{
173 int retval;
174
175 retval = 0;
176
177 do {
178 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
179 || ((paddr & (dmat->alignment - 1)) != 0))
180 && (dmat->filter == NULL
181 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
182 retval = 1;
183
184 dmat = dmat->parent;
185 } while (retval == 0 && dmat != NULL);
186 return (retval);
187}
188
189/*
190 * Convenience function for manipulating driver locks from busdma (during
191 * busdma_swi, for example). Drivers that don't provide their own locks
192 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
193 * non-mutex locking scheme don't have to use this at all.
194 */
195void
196busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
197{
198 struct mtx *dmtx;
199
200 dmtx = (struct mtx *)arg;
201 switch (op) {
202 case BUS_DMA_LOCK:
203 mtx_lock(dmtx);
204 break;
205 case BUS_DMA_UNLOCK:
206 mtx_unlock(dmtx);
207 break;
208 default:
209 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
210 }
211}
212
213/*
214 * dflt_lock should never get called. It gets put into the dma tag when
215 * lockfunc == NULL, which is only valid if the maps that are associated
216 * with the tag are meant to never be defered.
217 * XXX Should have a way to identify which driver is responsible here.
218 */
219static void
220dflt_lock(void *arg, bus_dma_lock_op_t op)
221{
222 panic("driver error: busdma dflt_lock called");
223}
224
225/*
226 * Allocate a device specific dma_tag.
227 */
151 * Allocate a device specific dma_tag.
152 */
228int
229bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
230 bus_addr_t boundary, bus_addr_t lowaddr,
231 bus_addr_t highaddr, bus_dma_filter_t *filter,
232 void *filterarg, bus_size_t maxsize, int nsegments,
233 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
234 void *lockfuncarg, bus_dma_tag_t *dmat)
153static int
154bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
155 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
156 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
157 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
158 void *lockfuncarg, bus_dma_tag_t *dmat)
235{
236 bus_dma_tag_t newtag;
159{
160 bus_dma_tag_t newtag;
237 int error = 0;
161 int error;
238
162
239 /* Basic sanity checking */
240 if (boundary != 0 && boundary < maxsegsz)
241 maxsegsz = boundary;
242
243 if (maxsegsz == 0) {
244 return (EINVAL);
245 }
246
247 /* Return a NULL tag on failure */
248 *dmat = NULL;
163 *dmat = NULL;
164 error = common_bus_dma_tag_create(parent != NULL ? &parent->common :
165 NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg,
166 maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
167 sizeof (struct bus_dma_tag), (void **)&newtag);
168 if (error != 0)
169 return (error);
249
170
250 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
251 M_ZERO | M_NOWAIT);
252 if (newtag == NULL) {
253 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
254 __func__, newtag, 0, ENOMEM);
255 return (ENOMEM);
256 }
257
258 newtag->parent = parent;
259 newtag->alignment = alignment;
260 newtag->boundary = boundary;
261 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
262 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
263 newtag->filter = filter;
264 newtag->filterarg = filterarg;
265 newtag->maxsize = maxsize;
266 newtag->nsegments = nsegments;
267 newtag->maxsegsz = maxsegsz;
268 newtag->flags = flags;
269 newtag->ref_count = 1; /* Count ourself */
171 newtag->common.impl = &bus_dma_bounce_impl;
270 newtag->map_count = 0;
172 newtag->map_count = 0;
271 if (lockfunc != NULL) {
272 newtag->lockfunc = lockfunc;
273 newtag->lockfuncarg = lockfuncarg;
274 } else {
275 newtag->lockfunc = dflt_lock;
276 newtag->lockfuncarg = NULL;
277 }
278 newtag->segments = NULL;
279
173 newtag->segments = NULL;
174
280 /* Take into account any restrictions imposed by our parent tag */
281 if (parent != NULL) {
282 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
283 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
284 if (newtag->boundary == 0)
285 newtag->boundary = parent->boundary;
286 else if (parent->boundary != 0)
287 newtag->boundary = MIN(parent->boundary,
288 newtag->boundary);
289 if ((newtag->filter != NULL) ||
290 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
291 newtag->flags |= BUS_DMA_COULD_BOUNCE;
292 if (newtag->filter == NULL) {
293 /*
294 * Short circuit looking at our parent directly
295 * since we have encapsulated all of its information
296 */
297 newtag->filter = parent->filter;
298 newtag->filterarg = parent->filterarg;
299 newtag->parent = parent->parent;
300 }
301 if (newtag->parent != NULL)
302 atomic_add_int(&parent->ref_count, 1);
303 }
175 if (parent != NULL && ((newtag->common.filter != NULL) ||
176 ((parent->common.flags & BUS_DMA_COULD_BOUNCE) != 0)))
177 newtag->common.flags |= BUS_DMA_COULD_BOUNCE;
304
178
305 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
306 || newtag->alignment > 1)
307 newtag->flags |= BUS_DMA_COULD_BOUNCE;
179 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) ||
180 newtag->common.alignment > 1)
181 newtag->common.flags |= BUS_DMA_COULD_BOUNCE;
308
182
309 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
183 if (((newtag->common.flags & BUS_DMA_COULD_BOUNCE) != 0) &&
310 (flags & BUS_DMA_ALLOCNOW) != 0) {
311 struct bounce_zone *bz;
312
313 /* Must bounce */
184 (flags & BUS_DMA_ALLOCNOW) != 0) {
185 struct bounce_zone *bz;
186
187 /* Must bounce */
314
315 if ((error = alloc_bounce_zone(newtag)) != 0) {
316 free(newtag, M_DEVBUF);
317 return (error);
318 }
319 bz = newtag->bounce_zone;
320
321 if (ptoa(bz->total_bpages) < maxsize) {
322 int pages;
323
324 pages = atop(maxsize) - bz->total_bpages;
325
326 /* Add pages to our bounce pool */
327 if (alloc_bounce_pages(newtag, pages) < pages)
328 error = ENOMEM;
329 }
330 /* Performed initial allocation */
188 if ((error = alloc_bounce_zone(newtag)) != 0) {
189 free(newtag, M_DEVBUF);
190 return (error);
191 }
192 bz = newtag->bounce_zone;
193
194 if (ptoa(bz->total_bpages) < maxsize) {
195 int pages;
196
197 pages = atop(maxsize) - bz->total_bpages;
198
199 /* Add pages to our bounce pool */
200 if (alloc_bounce_pages(newtag, pages) < pages)
201 error = ENOMEM;
202 }
203 /* Performed initial allocation */
331 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
332 }
204 newtag->common.flags |= BUS_DMA_MIN_ALLOC_COMP;
205 } else
206 error = 0;
333
207
334 if (error != 0) {
208 if (error != 0)
335 free(newtag, M_DEVBUF);
209 free(newtag, M_DEVBUF);
336 } else {
210 else
337 *dmat = newtag;
211 *dmat = newtag;
338 }
339 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
212 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
340 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
213 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
214 error);
341 return (error);
342}
343
215 return (error);
216}
217
344int
345bus_dma_tag_destroy(bus_dma_tag_t dmat)
218static int
219bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
346{
220{
347 bus_dma_tag_t dmat_copy;
221 bus_dma_tag_t dmat_copy, parent;
348 int error;
349
350 error = 0;
351 dmat_copy = dmat;
352
353 if (dmat != NULL) {
222 int error;
223
224 error = 0;
225 dmat_copy = dmat;
226
227 if (dmat != NULL) {
354
355 if (dmat->map_count != 0) {
356 error = EBUSY;
357 goto out;
358 }
228 if (dmat->map_count != 0) {
229 error = EBUSY;
230 goto out;
231 }
359
360 while (dmat != NULL) {
232 while (dmat != NULL) {
361 bus_dma_tag_t parent;
362
363 parent = dmat->parent;
364 atomic_subtract_int(&dmat->ref_count, 1);
365 if (dmat->ref_count == 0) {
233 parent = (bus_dma_tag_t)dmat->common.parent;
234 atomic_subtract_int(&dmat->common.ref_count, 1);
235 if (dmat->common.ref_count == 0) {
366 if (dmat->segments != NULL)
367 free(dmat->segments, M_DEVBUF);
368 free(dmat, M_DEVBUF);
369 /*
370 * Last reference count, so
371 * release our reference
372 * count on our parent.
373 */

--- 6 unchanged lines hidden (view full) ---

380 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
381 return (error);
382}
383
384/*
385 * Allocate a handle for mapping from kva/uva/physical
386 * address space into bus device space.
387 */
236 if (dmat->segments != NULL)
237 free(dmat->segments, M_DEVBUF);
238 free(dmat, M_DEVBUF);
239 /*
240 * Last reference count, so
241 * release our reference
242 * count on our parent.
243 */

--- 6 unchanged lines hidden (view full) ---

250 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
251 return (error);
252}
253
254/*
255 * Allocate a handle for mapping from kva/uva/physical
256 * address space into bus device space.
257 */
388int
389bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
258static int
259bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
390{
260{
391 int error;
261 struct bounce_zone *bz;
262 int error, maxpages, pages;
392
393 error = 0;
394
395 if (dmat->segments == NULL) {
396 dmat->segments = (bus_dma_segment_t *)malloc(
263
264 error = 0;
265
266 if (dmat->segments == NULL) {
267 dmat->segments = (bus_dma_segment_t *)malloc(
397 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
398 M_NOWAIT);
268 sizeof(bus_dma_segment_t) * dmat->common.nsegments,
269 M_DEVBUF, M_NOWAIT);
399 if (dmat->segments == NULL) {
400 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
401 __func__, dmat, ENOMEM);
402 return (ENOMEM);
403 }
404 }
405
406 /*
407 * Bouncing might be required if the driver asks for an active
408 * exclusion region, a data alignment that is stricter than 1, and/or
409 * an active address boundary.
410 */
270 if (dmat->segments == NULL) {
271 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
272 __func__, dmat, ENOMEM);
273 return (ENOMEM);
274 }
275 }
276
277 /*
278 * Bouncing might be required if the driver asks for an active
279 * exclusion region, a data alignment that is stricter than 1, and/or
280 * an active address boundary.
281 */
411 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
412
282 if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) {
413 /* Must bounce */
283 /* Must bounce */
414 struct bounce_zone *bz;
415 int maxpages;
416
417 if (dmat->bounce_zone == NULL) {
418 if ((error = alloc_bounce_zone(dmat)) != 0)
419 return (error);
420 }
421 bz = dmat->bounce_zone;
422
423 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
284 if (dmat->bounce_zone == NULL) {
285 if ((error = alloc_bounce_zone(dmat)) != 0)
286 return (error);
287 }
288 bz = dmat->bounce_zone;
289
290 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
424 M_NOWAIT | M_ZERO);
291 M_NOWAIT | M_ZERO);
425 if (*mapp == NULL) {
426 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
427 __func__, dmat, ENOMEM);
428 return (ENOMEM);
429 }
430
431 /* Initialize the new map */
432 STAILQ_INIT(&((*mapp)->bpages));
433
434 /*
435 * Attempt to add pages to our pool on a per-instance
436 * basis up to a sane limit.
437 */
292 if (*mapp == NULL) {
293 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
294 __func__, dmat, ENOMEM);
295 return (ENOMEM);
296 }
297
298 /* Initialize the new map */
299 STAILQ_INIT(&((*mapp)->bpages));
300
301 /*
302 * Attempt to add pages to our pool on a per-instance
303 * basis up to a sane limit.
304 */
438 if (dmat->alignment > 1)
305 if (dmat->common.alignment > 1)
439 maxpages = MAX_BPAGES;
440 else
306 maxpages = MAX_BPAGES;
307 else
441 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
442 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
443 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
444 int pages;
445
446 pages = MAX(atop(dmat->maxsize), 1);
308 maxpages = MIN(MAX_BPAGES, Maxmem -
309 atop(dmat->common.lowaddr));
310 if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP) == 0 ||
311 (bz->map_count > 0 && bz->total_bpages < maxpages)) {
312 pages = MAX(atop(dmat->common.maxsize), 1);
447 pages = MIN(maxpages - bz->total_bpages, pages);
448 pages = MAX(pages, 1);
449 if (alloc_bounce_pages(dmat, pages) < pages)
450 error = ENOMEM;
313 pages = MIN(maxpages - bz->total_bpages, pages);
314 pages = MAX(pages, 1);
315 if (alloc_bounce_pages(dmat, pages) < pages)
316 error = ENOMEM;
451
452 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
453 if (error == 0)
454 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
455 } else {
317 if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP)
318 == 0) {
319 if (error == 0) {
320 dmat->common.flags |=
321 BUS_DMA_MIN_ALLOC_COMP;
322 }
323 } else
456 error = 0;
324 error = 0;
457 }
458 }
459 bz->map_count++;
460 } else {
461 *mapp = NULL;
462 }
463 if (error == 0)
464 dmat->map_count++;
465 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
325 }
326 bz->map_count++;
327 } else {
328 *mapp = NULL;
329 }
330 if (error == 0)
331 dmat->map_count++;
332 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
466 __func__, dmat, dmat->flags, error);
333 __func__, dmat, dmat->common.flags, error);
467 return (error);
468}
469
470/*
471 * Destroy a handle for mapping from kva/uva/physical
472 * address space into bus device space.
473 */
334 return (error);
335}
336
337/*
338 * Destroy a handle for mapping from kva/uva/physical
339 * address space into bus device space.
340 */
474int
475bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
341static int
342bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
476{
343{
344
477 if (map != NULL && map != &nobounce_dmamap && map != &contig_dmamap) {
478 if (STAILQ_FIRST(&map->bpages) != NULL) {
479 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
480 __func__, dmat, EBUSY);
481 return (EBUSY);
482 }
483 if (dmat->bounce_zone)
484 dmat->bounce_zone->map_count--;

--- 5 unchanged lines hidden (view full) ---

490}
491
492
493/*
494 * Allocate a piece of memory that can be efficiently mapped into
495 * bus device space based on the constraints lited in the dma tag.
496 * A dmamap to for use with dmamap_load is also allocated.
497 */
345 if (map != NULL && map != &nobounce_dmamap && map != &contig_dmamap) {
346 if (STAILQ_FIRST(&map->bpages) != NULL) {
347 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
348 __func__, dmat, EBUSY);
349 return (EBUSY);
350 }
351 if (dmat->bounce_zone)
352 dmat->bounce_zone->map_count--;

--- 5 unchanged lines hidden (view full) ---

358}
359
360
361/*
362 * Allocate a piece of memory that can be efficiently mapped into
363 * bus device space based on the constraints lited in the dma tag.
364 * A dmamap to for use with dmamap_load is also allocated.
365 */
498int
499bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
500 bus_dmamap_t *mapp)
366static int
367bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
368 bus_dmamap_t *mapp)
501{
502 vm_memattr_t attr;
503 int mflags;
504
505 if (flags & BUS_DMA_NOWAIT)
506 mflags = M_NOWAIT;
507 else
508 mflags = M_WAITOK;
509
510 /* If we succeed, no mapping/bouncing will be required */
511 *mapp = NULL;
512
513 if (dmat->segments == NULL) {
514 dmat->segments = (bus_dma_segment_t *)malloc(
369{
370 vm_memattr_t attr;
371 int mflags;
372
373 if (flags & BUS_DMA_NOWAIT)
374 mflags = M_NOWAIT;
375 else
376 mflags = M_WAITOK;
377
378 /* If we succeed, no mapping/bouncing will be required */
379 *mapp = NULL;
380
381 if (dmat->segments == NULL) {
382 dmat->segments = (bus_dma_segment_t *)malloc(
515 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
516 mflags);
383 sizeof(bus_dma_segment_t) * dmat->common.nsegments,
384 M_DEVBUF, mflags);
517 if (dmat->segments == NULL) {
518 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
385 if (dmat->segments == NULL) {
386 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
519 __func__, dmat, dmat->flags, ENOMEM);
387 __func__, dmat, dmat->common.flags, ENOMEM);
520 return (ENOMEM);
521 }
522 }
523 if (flags & BUS_DMA_ZERO)
524 mflags |= M_ZERO;
525 if (flags & BUS_DMA_NOCACHE)
526 attr = VM_MEMATTR_UNCACHEABLE;
527 else
528 attr = VM_MEMATTR_DEFAULT;
529
530 /*
531 * XXX:
532 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
533 * alignment guarantees of malloc need to be nailed down, and the
534 * code below should be rewritten to take that into account.
535 *
536 * In the meantime, we'll warn the user if malloc gets it wrong.
537 */
388 return (ENOMEM);
389 }
390 }
391 if (flags & BUS_DMA_ZERO)
392 mflags |= M_ZERO;
393 if (flags & BUS_DMA_NOCACHE)
394 attr = VM_MEMATTR_UNCACHEABLE;
395 else
396 attr = VM_MEMATTR_DEFAULT;
397
398 /*
399 * XXX:
400 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
401 * alignment guarantees of malloc need to be nailed down, and the
402 * code below should be rewritten to take that into account.
403 *
404 * In the meantime, we'll warn the user if malloc gets it wrong.
405 */
538 if ((dmat->maxsize <= PAGE_SIZE) &&
539 (dmat->alignment < dmat->maxsize) &&
540 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
406 if ((dmat->common.maxsize <= PAGE_SIZE) &&
407 (dmat->common.alignment < dmat->common.maxsize) &&
408 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
541 attr == VM_MEMATTR_DEFAULT) {
409 attr == VM_MEMATTR_DEFAULT) {
542 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
543 } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
544 dmat->alignment <= PAGE_SIZE &&
545 (dmat->boundary == 0 || dmat->boundary >= dmat->lowaddr)) {
410 *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags);
411 } else if (dmat->common.nsegments >= btoc(dmat->common.maxsize) &&
412 dmat->common.alignment <= PAGE_SIZE &&
413 (dmat->common.boundary == 0 ||
414 dmat->common.boundary >= dmat->common.lowaddr)) {
546 /* Page-based multi-segment allocations allowed */
415 /* Page-based multi-segment allocations allowed */
547 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
548 mflags, 0ul, dmat->lowaddr, attr);
416 *vaddr = (void *)kmem_alloc_attr(kernel_arena,
417 dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
418 attr);
549 *mapp = &contig_dmamap;
550 } else {
419 *mapp = &contig_dmamap;
420 } else {
551 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
552 mflags, 0ul, dmat->lowaddr, dmat->alignment ?
553 dmat->alignment : 1ul, dmat->boundary, attr);
421 *vaddr = (void *)kmem_alloc_contig(kernel_arena,
422 dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
423 dmat->common.alignment != 0 ? dmat->common.alignment : 1ul,
424 dmat->common.boundary, attr);
554 *mapp = &contig_dmamap;
555 }
556 if (*vaddr == NULL) {
557 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
425 *mapp = &contig_dmamap;
426 }
427 if (*vaddr == NULL) {
428 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
558 __func__, dmat, dmat->flags, ENOMEM);
429 __func__, dmat, dmat->common.flags, ENOMEM);
559 return (ENOMEM);
430 return (ENOMEM);
560 } else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
431 } else if (vtophys(*vaddr) & (dmat->common.alignment - 1)) {
561 printf("bus_dmamem_alloc failed to align memory properly.\n");
562 }
563 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
432 printf("bus_dmamem_alloc failed to align memory properly.\n");
433 }
434 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
564 __func__, dmat, dmat->flags, 0);
435 __func__, dmat, dmat->common.flags, 0);
565 return (0);
566}
567
568/*
569 * Free a piece of memory and it's allociated dmamap, that was allocated
570 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
571 */
436 return (0);
437}
438
439/*
440 * Free a piece of memory and it's allociated dmamap, that was allocated
441 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
442 */
572void
573bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
443static void
444bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
574{
575 /*
576 * dmamem does not need to be bounced, so the map should be
577 * NULL if malloc() was used and contig_dmamap if
578 * kmem_alloc_contig() was used.
579 */
580 if (!(map == NULL || map == &contig_dmamap))
581 panic("bus_dmamem_free: Invalid map freed\n");
582 if (map == NULL)
583 free(vaddr, M_DEVBUF);
584 else
445{
446 /*
447 * dmamem does not need to be bounced, so the map should be
448 * NULL if malloc() was used and contig_dmamap if
449 * kmem_alloc_contig() was used.
450 */
451 if (!(map == NULL || map == &contig_dmamap))
452 panic("bus_dmamem_free: Invalid map freed\n");
453 if (map == NULL)
454 free(vaddr, M_DEVBUF);
455 else
585 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
586 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
456 kmem_free(kernel_arena, (vm_offset_t)vaddr,
457 dmat->common.maxsize);
458 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
459 dmat->common.flags);
587}
588
589static void
590_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
591 bus_size_t buflen, int flags)
592{
593 bus_addr_t curaddr;
594 bus_size_t sgsize;
595
596 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
597 /*
598 * Count the number of bounce pages
599 * needed in order to complete this transfer
600 */
601 curaddr = buf;
602 while (buflen != 0) {
460}
461
462static void
463_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
464 bus_size_t buflen, int flags)
465{
466 bus_addr_t curaddr;
467 bus_size_t sgsize;
468
469 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
470 /*
471 * Count the number of bounce pages
472 * needed in order to complete this transfer
473 */
474 curaddr = buf;
475 while (buflen != 0) {
603 sgsize = MIN(buflen, dmat->maxsegsz);
604 if (run_filter(dmat, curaddr)) {
476 sgsize = MIN(buflen, dmat->common.maxsegsz);
477 if (bus_dma_run_filter(&dmat->common, curaddr)) {
605 sgsize = MIN(sgsize, PAGE_SIZE);
606 map->pagesneeded++;
607 }
608 curaddr += sgsize;
609 buflen -= sgsize;
610 }
611 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
612 }
613}
614
615static void
616_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
617 void *buf, bus_size_t buflen, int flags)
618{
619 vm_offset_t vaddr;
620 vm_offset_t vendaddr;
621 bus_addr_t paddr;
478 sgsize = MIN(sgsize, PAGE_SIZE);
479 map->pagesneeded++;
480 }
481 curaddr += sgsize;
482 buflen -= sgsize;
483 }
484 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
485 }
486}
487
488static void
489_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
490 void *buf, bus_size_t buflen, int flags)
491{
492 vm_offset_t vaddr;
493 vm_offset_t vendaddr;
494 bus_addr_t paddr;
495 bus_size_t sg_len;
622
623 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
624 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
496
497 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
498 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
625 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
626 dmat->boundary, dmat->alignment);
499 "alignment= %d", dmat->common.lowaddr,
500 ptoa((vm_paddr_t)Maxmem),
501 dmat->common.boundary, dmat->common.alignment);
627 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
628 map, &nobounce_dmamap, map->pagesneeded);
629 /*
630 * Count the number of bounce pages
631 * needed in order to complete this transfer
632 */
633 vaddr = (vm_offset_t)buf;
634 vendaddr = (vm_offset_t)buf + buflen;
635
636 while (vaddr < vendaddr) {
502 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
503 map, &nobounce_dmamap, map->pagesneeded);
504 /*
505 * Count the number of bounce pages
506 * needed in order to complete this transfer
507 */
508 vaddr = (vm_offset_t)buf;
509 vendaddr = (vm_offset_t)buf + buflen;
510
511 while (vaddr < vendaddr) {
637 bus_size_t sg_len;
638
639 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
640 if (pmap == kernel_pmap)
641 paddr = pmap_kextract(vaddr);
642 else
643 paddr = pmap_extract(pmap, vaddr);
512 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
513 if (pmap == kernel_pmap)
514 paddr = pmap_kextract(vaddr);
515 else
516 paddr = pmap_extract(pmap, vaddr);
644 if (run_filter(dmat, paddr) != 0) {
645 sg_len = roundup2(sg_len, dmat->alignment);
517 if (bus_dma_run_filter(&dmat->common, paddr) != 0) {
518 sg_len = roundup2(sg_len,
519 dmat->common.alignment);
646 map->pagesneeded++;
647 }
648 vaddr += sg_len;
649 }
650 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
651 }
652}
653

--- 21 unchanged lines hidden (view full) ---

675 return (0);
676}
677
678/*
679 * Add a single contiguous physical range to the segment list.
680 */
681static int
682_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
520 map->pagesneeded++;
521 }
522 vaddr += sg_len;
523 }
524 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
525 }
526}
527

--- 21 unchanged lines hidden (view full) ---

549 return (0);
550}
551
552/*
553 * Add a single contiguous physical range to the segment list.
554 */
555static int
556_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
683 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
557 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
684{
685 bus_addr_t baddr, bmask;
686 int seg;
687
688 /*
689 * Make sure we don't cross any boundaries.
690 */
558{
559 bus_addr_t baddr, bmask;
560 int seg;
561
562 /*
563 * Make sure we don't cross any boundaries.
564 */
691 bmask = ~(dmat->boundary - 1);
692 if (dmat->boundary > 0) {
693 baddr = (curaddr + dmat->boundary) & bmask;
565 bmask = ~(dmat->common.boundary - 1);
566 if (dmat->common.boundary > 0) {
567 baddr = (curaddr + dmat->common.boundary) & bmask;
694 if (sgsize > (baddr - curaddr))
695 sgsize = (baddr - curaddr);
696 }
697
698 /*
699 * Insert chunk into a segment, coalescing with
700 * previous segment if possible.
701 */
702 seg = *segp;
703 if (seg == -1) {
704 seg = 0;
705 segs[seg].ds_addr = curaddr;
706 segs[seg].ds_len = sgsize;
707 } else {
708 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
568 if (sgsize > (baddr - curaddr))
569 sgsize = (baddr - curaddr);
570 }
571
572 /*
573 * Insert chunk into a segment, coalescing with
574 * previous segment if possible.
575 */
576 seg = *segp;
577 if (seg == -1) {
578 seg = 0;
579 segs[seg].ds_addr = curaddr;
580 segs[seg].ds_len = sgsize;
581 } else {
582 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
709 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
710 (dmat->boundary == 0 ||
583 (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz &&
584 (dmat->common.boundary == 0 ||
711 (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
712 segs[seg].ds_len += sgsize;
713 else {
585 (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
586 segs[seg].ds_len += sgsize;
587 else {
714 if (++seg >= dmat->nsegments)
588 if (++seg >= dmat->common.nsegments)
715 return (0);
716 segs[seg].ds_addr = curaddr;
717 segs[seg].ds_len = sgsize;
718 }
719 }
720 *segp = seg;
721 return (sgsize);
722}
723
724/*
725 * Utility function to load a physical buffer. segp contains
726 * the starting segment on entrace, and the ending segment on exit.
727 */
589 return (0);
590 segs[seg].ds_addr = curaddr;
591 segs[seg].ds_len = sgsize;
592 }
593 }
594 *segp = seg;
595 return (sgsize);
596}
597
598/*
599 * Utility function to load a physical buffer. segp contains
600 * the starting segment on entrace, and the ending segment on exit.
601 */
728int
729_bus_dmamap_load_phys(bus_dma_tag_t dmat,
730 bus_dmamap_t map,
731 vm_paddr_t buf, bus_size_t buflen,
732 int flags,
733 bus_dma_segment_t *segs,
734 int *segp)
602static int
603bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
604 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
605 int *segp)
735{
736 bus_size_t sgsize;
737 bus_addr_t curaddr;
738 int error;
739
740 if (map == NULL || map == &contig_dmamap)
741 map = &nobounce_dmamap;
742
743 if (segs == NULL)
744 segs = dmat->segments;
745
606{
607 bus_size_t sgsize;
608 bus_addr_t curaddr;
609 int error;
610
611 if (map == NULL || map == &contig_dmamap)
612 map = &nobounce_dmamap;
613
614 if (segs == NULL)
615 segs = dmat->segments;
616
746 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
617 if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) {
747 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
748 if (map->pagesneeded != 0) {
749 error = _bus_dmamap_reserve_pages(dmat, map, flags);
750 if (error)
751 return (error);
752 }
753 }
754
755 while (buflen > 0) {
756 curaddr = buf;
618 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
619 if (map->pagesneeded != 0) {
620 error = _bus_dmamap_reserve_pages(dmat, map, flags);
621 if (error)
622 return (error);
623 }
624 }
625
626 while (buflen > 0) {
627 curaddr = buf;
757 sgsize = MIN(buflen, dmat->maxsegsz);
758 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
759 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
628 sgsize = MIN(buflen, dmat->common.maxsegsz);
629 if (((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) &&
630 map->pagesneeded != 0 &&
631 bus_dma_run_filter(&dmat->common, curaddr)) {
760 sgsize = MIN(sgsize, PAGE_SIZE);
761 curaddr = add_bounce_page(dmat, map, 0, curaddr,
632 sgsize = MIN(sgsize, PAGE_SIZE);
633 curaddr = add_bounce_page(dmat, map, 0, curaddr,
762 sgsize);
634 sgsize);
763 }
764 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
765 segp);
766 if (sgsize == 0)
767 break;
768 buf += sgsize;
769 buflen -= sgsize;
770 }
771
772 /*
773 * Did we fit?
774 */
775 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
776}
777
778/*
779 * Utility function to load a linear buffer. segp contains
780 * the starting segment on entrace, and the ending segment on exit.
781 */
635 }
636 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
637 segp);
638 if (sgsize == 0)
639 break;
640 buf += sgsize;
641 buflen -= sgsize;
642 }
643
644 /*
645 * Did we fit?
646 */
647 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
648}
649
650/*
651 * Utility function to load a linear buffer. segp contains
652 * the starting segment on entrace, and the ending segment on exit.
653 */
782int
783_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
784 bus_dmamap_t map,
785 void *buf, bus_size_t buflen,
786 pmap_t pmap,
787 int flags,
788 bus_dma_segment_t *segs,
789 int *segp)
654static int
655bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
656 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
657 int *segp)
790{
658{
791 bus_size_t sgsize;
659 bus_size_t sgsize, max_sgsize;
792 bus_addr_t curaddr;
793 vm_offset_t vaddr;
794 int error;
795
796 if (map == NULL || map == &contig_dmamap)
797 map = &nobounce_dmamap;
798
799 if (segs == NULL)
800 segs = dmat->segments;
801
660 bus_addr_t curaddr;
661 vm_offset_t vaddr;
662 int error;
663
664 if (map == NULL || map == &contig_dmamap)
665 map = &nobounce_dmamap;
666
667 if (segs == NULL)
668 segs = dmat->segments;
669
802 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
670 if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) {
803 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
804 if (map->pagesneeded != 0) {
805 error = _bus_dmamap_reserve_pages(dmat, map, flags);
806 if (error)
807 return (error);
808 }
809 }
810
811 vaddr = (vm_offset_t)buf;
671 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
672 if (map->pagesneeded != 0) {
673 error = _bus_dmamap_reserve_pages(dmat, map, flags);
674 if (error)
675 return (error);
676 }
677 }
678
679 vaddr = (vm_offset_t)buf;
812
813 while (buflen > 0) {
680 while (buflen > 0) {
814 bus_size_t max_sgsize;
815
816 /*
817 * Get the physical address for this segment.
818 */
819 if (pmap == kernel_pmap)
820 curaddr = pmap_kextract(vaddr);
821 else
822 curaddr = pmap_extract(pmap, vaddr);
823
824 /*
825 * Compute the segment size, and adjust counts.
826 */
681 /*
682 * Get the physical address for this segment.
683 */
684 if (pmap == kernel_pmap)
685 curaddr = pmap_kextract(vaddr);
686 else
687 curaddr = pmap_extract(pmap, vaddr);
688
689 /*
690 * Compute the segment size, and adjust counts.
691 */
827 max_sgsize = MIN(buflen, dmat->maxsegsz);
692 max_sgsize = MIN(buflen, dmat->common.maxsegsz);
828 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
693 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
829 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
830 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
831 sgsize = roundup2(sgsize, dmat->alignment);
694 if (((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) &&
695 map->pagesneeded != 0 &&
696 bus_dma_run_filter(&dmat->common, curaddr)) {
697 sgsize = roundup2(sgsize, dmat->common.alignment);
832 sgsize = MIN(sgsize, max_sgsize);
833 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
698 sgsize = MIN(sgsize, max_sgsize);
699 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
834 sgsize);
700 sgsize);
835 } else {
836 sgsize = MIN(sgsize, max_sgsize);
837 }
838 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
701 } else {
702 sgsize = MIN(sgsize, max_sgsize);
703 }
704 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
839 segp);
705 segp);
840 if (sgsize == 0)
841 break;
842 vaddr += sgsize;
843 buflen -= sgsize;
844 }
845
846 /*
847 * Did we fit?
848 */
849 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
850}
851
706 if (sgsize == 0)
707 break;
708 vaddr += sgsize;
709 buflen -= sgsize;
710 }
711
712 /*
713 * Did we fit?
714 */
715 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
716}
717
852int
853_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
854 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
855 bus_dma_segment_t *segs, int *segp)
718static void
719bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
720 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
856{
857
721{
722
858 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
859 segs, segp));
723 if (map == NULL)
724 return;
725 map->mem = *mem;
726 map->dmat = dmat;
727 map->callback = callback;
728 map->callback_arg = callback_arg;
860}
861
729}
730
862void
863__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
864 struct memdesc *mem, bus_dmamap_callback_t *callback,
865 void *callback_arg)
731static bus_dma_segment_t *
732bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
733 bus_dma_segment_t *segs, int nsegs, int error)
866{
734{
867 if (map != NULL) {
868 map->mem = *mem;
869 map->dmat = dmat;
870 map->callback = callback;
871 map->callback_arg = callback_arg;
872 }
873}
874
735
875bus_dma_segment_t *
876_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
877 bus_dma_segment_t *segs, int nsegs, int error)
878{
879
880 if (segs == NULL)
881 segs = dmat->segments;
882 return (segs);
883}
884
885/*
886 * Release the mapping held by map.
887 */
736 if (segs == NULL)
737 segs = dmat->segments;
738 return (segs);
739}
740
741/*
742 * Release the mapping held by map.
743 */
888void
889_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
744static void
745bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
890{
891 struct bounce_page *bpage;
892
893 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
894 STAILQ_REMOVE_HEAD(&map->bpages, links);
895 free_bounce_page(dmat, bpage);
896 }
897}
898
746{
747 struct bounce_page *bpage;
748
749 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
750 STAILQ_REMOVE_HEAD(&map->bpages, links);
751 free_bounce_page(dmat, bpage);
752 }
753}
754
899void
900_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
755static void
756bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
757 bus_dmasync_op_t op)
901{
902 struct bounce_page *bpage;
903
904 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
905 /*
906 * Handle data bouncing. We might also
907 * want to add support for invalidating
908 * the caches on broken hardware
909 */
910 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
758{
759 struct bounce_page *bpage;
760
761 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
762 /*
763 * Handle data bouncing. We might also
764 * want to add support for invalidating
765 * the caches on broken hardware
766 */
767 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
911 "performing bounce", __func__, dmat, dmat->flags, op);
768 "performing bounce", __func__, dmat,
769 dmat->common.flags, op);
912
770
913 if (op & BUS_DMASYNC_PREWRITE) {
771 if ((op & BUS_DMASYNC_PREWRITE) != 0) {
914 while (bpage != NULL) {
772 while (bpage != NULL) {
915 if (bpage->datavaddr != 0)
773 if (bpage->datavaddr != 0) {
916 bcopy((void *)bpage->datavaddr,
774 bcopy((void *)bpage->datavaddr,
917 (void *)bpage->vaddr,
918 bpage->datacount);
919 else
775 (void *)bpage->vaddr,
776 bpage->datacount);
777 } else {
920 physcopyout(bpage->dataaddr,
778 physcopyout(bpage->dataaddr,
921 (void *)bpage->vaddr,
922 bpage->datacount);
779 (void *)bpage->vaddr,
780 bpage->datacount);
781 }
923 bpage = STAILQ_NEXT(bpage, links);
924 }
925 dmat->bounce_zone->total_bounced++;
926 }
927
782 bpage = STAILQ_NEXT(bpage, links);
783 }
784 dmat->bounce_zone->total_bounced++;
785 }
786
928 if (op & BUS_DMASYNC_POSTREAD) {
787 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
929 while (bpage != NULL) {
788 while (bpage != NULL) {
930 if (bpage->datavaddr != 0)
789 if (bpage->datavaddr != 0) {
931 bcopy((void *)bpage->vaddr,
790 bcopy((void *)bpage->vaddr,
932 (void *)bpage->datavaddr,
933 bpage->datacount);
934 else
791 (void *)bpage->datavaddr,
792 bpage->datacount);
793 } else {
935 physcopyin((void *)bpage->vaddr,
794 physcopyin((void *)bpage->vaddr,
936 bpage->dataaddr,
937 bpage->datacount);
795 bpage->dataaddr,
796 bpage->datacount);
797 }
938 bpage = STAILQ_NEXT(bpage, links);
939 }
940 dmat->bounce_zone->total_bounced++;
941 }
942 }
943}
944
945static void

--- 29 unchanged lines hidden (view full) ---

975
976static int
977alloc_bounce_zone(bus_dma_tag_t dmat)
978{
979 struct bounce_zone *bz;
980
981 /* Check to see if we already have a suitable zone */
982 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
798 bpage = STAILQ_NEXT(bpage, links);
799 }
800 dmat->bounce_zone->total_bounced++;
801 }
802 }
803}
804
805static void

--- 29 unchanged lines hidden (view full) ---

835
836static int
837alloc_bounce_zone(bus_dma_tag_t dmat)
838{
839 struct bounce_zone *bz;
840
841 /* Check to see if we already have a suitable zone */
842 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
983 if ((dmat->alignment <= bz->alignment)
984 && (dmat->lowaddr >= bz->lowaddr)) {
843 if ((dmat->common.alignment <= bz->alignment) &&
844 (dmat->common.lowaddr >= bz->lowaddr)) {
985 dmat->bounce_zone = bz;
986 return (0);
987 }
988 }
989
990 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
991 M_NOWAIT | M_ZERO)) == NULL)
992 return (ENOMEM);
993
994 STAILQ_INIT(&bz->bounce_page_list);
995 bz->free_bpages = 0;
996 bz->reserved_bpages = 0;
997 bz->active_bpages = 0;
845 dmat->bounce_zone = bz;
846 return (0);
847 }
848 }
849
850 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
851 M_NOWAIT | M_ZERO)) == NULL)
852 return (ENOMEM);
853
854 STAILQ_INIT(&bz->bounce_page_list);
855 bz->free_bpages = 0;
856 bz->reserved_bpages = 0;
857 bz->active_bpages = 0;
998 bz->lowaddr = dmat->lowaddr;
999 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
858 bz->lowaddr = dmat->common.lowaddr;
859 bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE);
1000 bz->map_count = 0;
1001 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1002 busdma_zonecount++;
1003 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1004 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1005 dmat->bounce_zone = bz;
1006
1007 sysctl_ctx_init(&bz->sysctl_tree);

--- 121 unchanged lines hidden (view full) ---

1129 if (bpage == NULL)
1130 panic("add_bounce_page: free page list is empty");
1131
1132 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1133 bz->reserved_bpages--;
1134 bz->active_bpages++;
1135 mtx_unlock(&bounce_lock);
1136
860 bz->map_count = 0;
861 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
862 busdma_zonecount++;
863 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
864 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
865 dmat->bounce_zone = bz;
866
867 sysctl_ctx_init(&bz->sysctl_tree);

--- 121 unchanged lines hidden (view full) ---

989 if (bpage == NULL)
990 panic("add_bounce_page: free page list is empty");
991
992 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
993 bz->reserved_bpages--;
994 bz->active_bpages++;
995 mtx_unlock(&bounce_lock);
996
1137 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
997 if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1138 /* Page offset needs to be preserved. */
1139 bpage->vaddr |= vaddr & PAGE_MASK;
1140 bpage->busaddr |= vaddr & PAGE_MASK;
1141 }
1142 bpage->datavaddr = vaddr;
1143 bpage->dataaddr = addr;
1144 bpage->datacount = size;
1145 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);

--- 4 unchanged lines hidden (view full) ---

1150free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1151{
1152 struct bus_dmamap *map;
1153 struct bounce_zone *bz;
1154
1155 bz = dmat->bounce_zone;
1156 bpage->datavaddr = 0;
1157 bpage->datacount = 0;
998 /* Page offset needs to be preserved. */
999 bpage->vaddr |= vaddr & PAGE_MASK;
1000 bpage->busaddr |= vaddr & PAGE_MASK;
1001 }
1002 bpage->datavaddr = vaddr;
1003 bpage->dataaddr = addr;
1004 bpage->datacount = size;
1005 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);

--- 4 unchanged lines hidden (view full) ---

1010free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1011{
1012 struct bus_dmamap *map;
1013 struct bounce_zone *bz;
1014
1015 bz = dmat->bounce_zone;
1016 bpage->datavaddr = 0;
1017 bpage->datacount = 0;
1158 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1018 if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1159 /*
1160 * Reset the bounce page to start at offset 0. Other uses
1161 * of this bounce page may need to store a full page of
1162 * data and/or assume it starts on a page boundary.
1163 */
1164 bpage->vaddr &= ~PAGE_MASK;
1165 bpage->busaddr &= ~PAGE_MASK;
1166 }
1167
1168 mtx_lock(&bounce_lock);
1169 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1170 bz->free_bpages++;
1171 bz->active_bpages--;
1172 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1173 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1174 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1175 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1019 /*
1020 * Reset the bounce page to start at offset 0. Other uses
1021 * of this bounce page may need to store a full page of
1022 * data and/or assume it starts on a page boundary.
1023 */
1024 bpage->vaddr &= ~PAGE_MASK;
1025 bpage->busaddr &= ~PAGE_MASK;
1026 }
1027
1028 mtx_lock(&bounce_lock);
1029 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1030 bz->free_bpages++;
1031 bz->active_bpages--;
1032 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1033 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1034 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1035 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1176 map, links);
1036 map, links);
1177 busdma_swi_pending = 1;
1178 bz->total_deferred++;
1179 swi_sched(vm_ih, 0);
1180 }
1181 }
1182 mtx_unlock(&bounce_lock);
1183}
1184
1185void
1186busdma_swi(void)
1187{
1188 bus_dma_tag_t dmat;
1189 struct bus_dmamap *map;
1190
1191 mtx_lock(&bounce_lock);
1192 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1193 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1194 mtx_unlock(&bounce_lock);
1195 dmat = map->dmat;
1037 busdma_swi_pending = 1;
1038 bz->total_deferred++;
1039 swi_sched(vm_ih, 0);
1040 }
1041 }
1042 mtx_unlock(&bounce_lock);
1043}
1044
1045void
1046busdma_swi(void)
1047{
1048 bus_dma_tag_t dmat;
1049 struct bus_dmamap *map;
1050
1051 mtx_lock(&bounce_lock);
1052 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1053 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1054 mtx_unlock(&bounce_lock);
1055 dmat = map->dmat;
1196 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1056 (dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_LOCK);
1197 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1057 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1198 map->callback, map->callback_arg,
1199 BUS_DMA_WAITOK);
1200 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1058 map->callback, map->callback_arg, BUS_DMA_WAITOK);
1059 (dmat->common.lockfunc)(dmat->common.lockfuncarg,
1060 BUS_DMA_UNLOCK);
1201 mtx_lock(&bounce_lock);
1202 }
1203 mtx_unlock(&bounce_lock);
1204}
1061 mtx_lock(&bounce_lock);
1062 }
1063 mtx_unlock(&bounce_lock);
1064}
1065
1066struct bus_dma_impl bus_dma_bounce_impl = {
1067 .tag_create = bounce_bus_dma_tag_create,
1068 .tag_destroy = bounce_bus_dma_tag_destroy,
1069 .map_create = bounce_bus_dmamap_create,
1070 .map_destroy = bounce_bus_dmamap_destroy,
1071 .mem_alloc = bounce_bus_dmamem_alloc,
1072 .mem_free = bounce_bus_dmamem_free,
1073 .load_phys = bounce_bus_dmamap_load_phys,
1074 .load_buffer = bounce_bus_dmamap_load_buffer,
1075 .load_ma = bus_dmamap_load_ma_triv,
1076 .map_waitok = bounce_bus_dmamap_waitok,
1077 .map_complete = bounce_bus_dmamap_complete,
1078 .map_unload = bounce_bus_dmamap_unload,
1079 .map_sync = bounce_bus_dmamap_sync
1080};