busdma_machdep-v6.c revision 244469
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2010 Mark Tinguely 4 * Copyright (c) 2004 Olivier Houchard 5 * Copyright (c) 2002 Peter Grehan 6 * Copyright (c) 1997, 1998 Justin T. Gibbs. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions, and the following disclaimer, 14 * without modification, immediately at the beginning of the file. 15 * 2. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 22 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep-v6.c 244469 2012-12-20 00:35:26Z cognet $"); 35 36#define _ARM32_BUS_DMA_PRIVATE 37#include <sys/param.h> 38#include <sys/kdb.h> 39#include <ddb/ddb.h> 40#include <ddb/db_output.h> 41#include <sys/systm.h> 42#include <sys/malloc.h> 43#include <sys/bus.h> 44#include <sys/busdma_bufalloc.h> 45#include <sys/interrupt.h> 46#include <sys/kernel.h> 47#include <sys/ktr.h> 48#include <sys/lock.h> 49#include <sys/proc.h> 50#include <sys/mutex.h> 51#include <sys/mbuf.h> 52#include <sys/uio.h> 53#include <sys/sysctl.h> 54 55#include <vm/vm.h> 56#include <vm/vm_page.h> 57#include <vm/vm_map.h> 58#include <vm/vm_extern.h> 59#include <vm/vm_kern.h> 60 61#include <machine/atomic.h> 62#include <machine/bus.h> 63#include <machine/cpufunc.h> 64#include <machine/md_var.h> 65 66#define MAX_BPAGES 64 67#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 68#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 69 70#define FIX_DMAP_BUS_DMASYNC_POSTREAD 71 72struct bounce_zone; 73 74struct bus_dma_tag { 75 bus_dma_tag_t parent; 76 bus_size_t alignment; 77 bus_size_t boundary; 78 bus_addr_t lowaddr; 79 bus_addr_t highaddr; 80 bus_dma_filter_t *filter; 81 void *filterarg; 82 bus_size_t maxsize; 83 u_int nsegments; 84 bus_size_t maxsegsz; 85 int flags; 86 int ref_count; 87 int map_count; 88 bus_dma_lock_t *lockfunc; 89 void *lockfuncarg; 90 struct bounce_zone *bounce_zone; 91 /* 92 * DMA range for this tag. If the page doesn't fall within 93 * one of these ranges, an error is returned. The caller 94 * may then decide what to do with the transfer. If the 95 * range pointer is NULL, it is ignored. 96 */ 97 struct arm32_dma_range *ranges; 98 int _nranges; 99 /* 100 * Most tags need one or two segments, and can use the local tagsegs 101 * array. For tags with a larger limit, we'll allocate a bigger array 102 * on first use. 103 */ 104 bus_dma_segment_t *segments; 105 bus_dma_segment_t tagsegs[2]; 106 107 108}; 109 110struct bounce_page { 111 vm_offset_t vaddr; /* kva of bounce buffer */ 112 bus_addr_t busaddr; /* Physical address */ 113 vm_offset_t datavaddr; /* kva of client data */ 114 bus_size_t datacount; /* client data count */ 115 STAILQ_ENTRY(bounce_page) links; 116}; 117 118struct sync_list { 119 vm_offset_t vaddr; /* kva of bounce buffer */ 120 bus_addr_t busaddr; /* Physical address */ 121 bus_size_t datacount; /* client data count */ 122 STAILQ_ENTRY(sync_list) slinks; 123}; 124 125int busdma_swi_pending; 126 127struct bounce_zone { 128 STAILQ_ENTRY(bounce_zone) links; 129 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 130 int total_bpages; 131 int free_bpages; 132 int reserved_bpages; 133 int active_bpages; 134 int total_bounced; 135 int total_deferred; 136 int map_count; 137 bus_size_t alignment; 138 bus_addr_t lowaddr; 139 char zoneid[8]; 140 char lowaddrid[20]; 141 struct sysctl_ctx_list sysctl_tree; 142 struct sysctl_oid *sysctl_tree_top; 143}; 144 145static struct mtx bounce_lock; 146static int total_bpages; 147static int busdma_zonecount; 148static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 149 150SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 151SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 152 "Total bounce pages"); 153 154struct bus_dmamap { 155 struct bp_list bpages; 156 int pagesneeded; 157 int pagesreserved; 158 bus_dma_tag_t dmat; 159 void *buf; /* unmapped buffer pointer */ 160 bus_size_t buflen; /* unmapped buffer length */ 161 pmap_t pmap; 162 bus_dmamap_callback_t *callback; 163 void *callback_arg; 164 int flags; 165#define DMAMAP_COHERENT (1 << 0) 166 STAILQ_ENTRY(bus_dmamap) links; 167 STAILQ_HEAD(,sync_list) slist; 168}; 169 170static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 171static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 172 173static void init_bounce_pages(void *dummy); 174static int alloc_bounce_zone(bus_dma_tag_t dmat); 175static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 176static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 177 int commit); 178static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 179 vm_offset_t vaddr, bus_size_t size); 180static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 181int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 182static int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 183 void *buf, bus_size_t buflen, int flags); 184 185static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 186static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 187static void 188busdma_init(void *dummy) 189{ 190 191 /* Create a cache of buffers in standard (cacheable) memory. */ 192 standard_allocator = busdma_bufalloc_create("buffer", 193 arm_dcache_align, /* minimum_alignment */ 194 NULL, /* uma_alloc func */ 195 NULL, /* uma_free func */ 196 0); /* uma_zcreate_flags */ 197 198 /* 199 * Create a cache of buffers in uncacheable memory, to implement the 200 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 201 */ 202 coherent_allocator = busdma_bufalloc_create("coherent", 203 arm_dcache_align, /* minimum_alignment */ 204 busdma_bufalloc_alloc_uncacheable, 205 busdma_bufalloc_free_uncacheable, 206 0); /* uma_zcreate_flags */ 207} 208 209/* 210 * This init historically used SI_SUB_VM, but now the init code requires 211 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 212 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 213 * SI_SUB_KMEM and SI_ORDER_THIRD. 214 */ 215SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 216 217static __inline int 218_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 219{ 220 int i; 221 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 222 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 223 || (lowaddr < phys_avail[i] && 224 highaddr > phys_avail[i])) 225 return (1); 226 } 227 return (0); 228} 229 230static __inline struct arm32_dma_range * 231_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 232 bus_addr_t curaddr) 233{ 234 struct arm32_dma_range *dr; 235 int i; 236 237 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 238 if (curaddr >= dr->dr_sysbase && 239 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 240 return (dr); 241 } 242 243 return (NULL); 244} 245 246/* 247 * Return true if a match is made. 248 * 249 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 250 * 251 * If paddr is within the bounds of the dma tag then call the filter callback 252 * to check for a match, if there is no filter callback then assume a match. 253 */ 254int 255run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 256{ 257 int retval; 258 259 retval = 0; 260 261 do { 262 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 263 || ((paddr & (dmat->alignment - 1)) != 0)) 264 && (dmat->filter == NULL 265 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 266 retval = 1; 267 268 dmat = dmat->parent; 269 } while (retval == 0 && dmat != NULL); 270 return (retval); 271} 272 273/* 274 * Convenience function for manipulating driver locks from busdma (during 275 * busdma_swi, for example). Drivers that don't provide their own locks 276 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 277 * non-mutex locking scheme don't have to use this at all. 278 */ 279void 280busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 281{ 282 struct mtx *dmtx; 283 284 dmtx = (struct mtx *)arg; 285 switch (op) { 286 case BUS_DMA_LOCK: 287 mtx_lock(dmtx); 288 break; 289 case BUS_DMA_UNLOCK: 290 mtx_unlock(dmtx); 291 break; 292 default: 293 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 294 } 295} 296 297/* 298 * dflt_lock should never get called. It gets put into the dma tag when 299 * lockfunc == NULL, which is only valid if the maps that are associated 300 * with the tag are meant to never be defered. 301 * XXX Should have a way to identify which driver is responsible here. 302 */ 303static void 304dflt_lock(void *arg, bus_dma_lock_op_t op) 305{ 306 panic("driver error: busdma dflt_lock called"); 307} 308 309/* 310 * Allocate a device specific dma_tag. 311 */ 312int 313bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 314 bus_size_t boundary, bus_addr_t lowaddr, 315 bus_addr_t highaddr, bus_dma_filter_t *filter, 316 void *filterarg, bus_size_t maxsize, int nsegments, 317 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 318 void *lockfuncarg, bus_dma_tag_t *dmat) 319{ 320 bus_dma_tag_t newtag; 321 int error = 0; 322 323#if 0 324 if (!parent) 325 parent = arm_root_dma_tag; 326#endif 327 328 /* Basic sanity checking */ 329 if (boundary != 0 && boundary < maxsegsz) 330 maxsegsz = boundary; 331 332 /* Return a NULL tag on failure */ 333 *dmat = NULL; 334 335 if (maxsegsz == 0) { 336 return (EINVAL); 337 } 338 339 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 340 M_ZERO | M_NOWAIT); 341 if (newtag == NULL) { 342 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 343 __func__, newtag, 0, error); 344 return (ENOMEM); 345 } 346 347 newtag->parent = parent; 348 newtag->alignment = alignment; 349 newtag->boundary = boundary; 350 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 351 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 352 (PAGE_SIZE - 1); 353 newtag->filter = filter; 354 newtag->filterarg = filterarg; 355 newtag->maxsize = maxsize; 356 newtag->nsegments = nsegments; 357 newtag->maxsegsz = maxsegsz; 358 newtag->flags = flags; 359 newtag->ref_count = 1; /* Count ourself */ 360 newtag->map_count = 0; 361 newtag->ranges = bus_dma_get_range(); 362 newtag->_nranges = bus_dma_get_range_nb(); 363 if (lockfunc != NULL) { 364 newtag->lockfunc = lockfunc; 365 newtag->lockfuncarg = lockfuncarg; 366 } else { 367 newtag->lockfunc = dflt_lock; 368 newtag->lockfuncarg = NULL; 369 } 370 /* 371 * If all the segments we need fit into the local tagsegs array, set the 372 * pointer now. Otherwise NULL the pointer and an array of segments 373 * will be allocated later, on first use. We don't pre-allocate now 374 * because some tags exist just to pass contraints to children in the 375 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 376 * sure don't want to try to allocate an array for that. 377 */ 378 if (newtag->nsegments <= nitems(newtag->tagsegs)) 379 newtag->segments = newtag->tagsegs; 380 else 381 newtag->segments = NULL; 382 383 /* Take into account any restrictions imposed by our parent tag */ 384 if (parent != NULL) { 385 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 386 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 387 if (newtag->boundary == 0) 388 newtag->boundary = parent->boundary; 389 else if (parent->boundary != 0) 390 newtag->boundary = MIN(parent->boundary, 391 newtag->boundary); 392 if ((newtag->filter != NULL) || 393 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 394 newtag->flags |= BUS_DMA_COULD_BOUNCE; 395 if (newtag->filter == NULL) { 396 /* 397 * Short circuit looking at our parent directly 398 * since we have encapsulated all of its information 399 */ 400 newtag->filter = parent->filter; 401 newtag->filterarg = parent->filterarg; 402 newtag->parent = parent->parent; 403 } 404 if (newtag->parent != NULL) 405 atomic_add_int(&parent->ref_count, 1); 406 } 407 408 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 409 || newtag->alignment > 1) 410 newtag->flags |= BUS_DMA_COULD_BOUNCE; 411 412 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 413 (flags & BUS_DMA_ALLOCNOW) != 0) { 414 struct bounce_zone *bz; 415 416 /* Must bounce */ 417 418 if ((error = alloc_bounce_zone(newtag)) != 0) { 419 free(newtag, M_DEVBUF); 420 return (error); 421 } 422 bz = newtag->bounce_zone; 423 424 if (ptoa(bz->total_bpages) < maxsize) { 425 int pages; 426 427 pages = atop(maxsize) - bz->total_bpages; 428 429 /* Add pages to our bounce pool */ 430 if (alloc_bounce_pages(newtag, pages) < pages) 431 error = ENOMEM; 432 } 433 /* Performed initial allocation */ 434 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 435 } else 436 newtag->bounce_zone = NULL; 437 438 if (error != 0) { 439 free(newtag, M_DEVBUF); 440 } else { 441 *dmat = newtag; 442 } 443 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 444 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 445 return (error); 446} 447 448int 449bus_dma_tag_destroy(bus_dma_tag_t dmat) 450{ 451 bus_dma_tag_t dmat_copy; 452 int error; 453 454 error = 0; 455 dmat_copy = dmat; 456 457 if (dmat != NULL) { 458 459 if (dmat->map_count != 0) { 460 error = EBUSY; 461 goto out; 462 } 463 464 while (dmat != NULL) { 465 bus_dma_tag_t parent; 466 467 parent = dmat->parent; 468 atomic_subtract_int(&dmat->ref_count, 1); 469 if (dmat->ref_count == 0) { 470 if (dmat->segments != NULL && 471 dmat->segments != dmat->tagsegs) 472 free(dmat->segments, M_DEVBUF); 473 free(dmat, M_DEVBUF); 474 /* 475 * Last reference count, so 476 * release our reference 477 * count on our parent. 478 */ 479 dmat = parent; 480 } else 481 dmat = NULL; 482 } 483 } 484out: 485 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 486 return (error); 487} 488 489/* 490 * Allocate a handle for mapping from kva/uva/physical 491 * address space into bus device space. 492 */ 493int 494bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 495{ 496 int error; 497 498 error = 0; 499 500 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 501 M_NOWAIT | M_ZERO); 502 if (*mapp == NULL) { 503 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 504 return (ENOMEM); 505 } 506 STAILQ_INIT(&((*mapp)->slist)); 507 508 if (dmat->segments == NULL) { 509 dmat->segments = (bus_dma_segment_t *)malloc( 510 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 511 M_NOWAIT); 512 if (dmat->segments == NULL) { 513 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 514 __func__, dmat, ENOMEM); 515 free(*mapp, M_DEVBUF); 516 *mapp = NULL; 517 return (ENOMEM); 518 } 519 } 520 /* 521 * Bouncing might be required if the driver asks for an active 522 * exclusion region, a data alignment that is stricter than 1, and/or 523 * an active address boundary. 524 */ 525 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 526 527 /* Must bounce */ 528 struct bounce_zone *bz; 529 int maxpages; 530 531 if (dmat->bounce_zone == NULL) { 532 if ((error = alloc_bounce_zone(dmat)) != 0) { 533 free(*mapp, M_DEVBUF); 534 *mapp = NULL; 535 return (error); 536 } 537 } 538 bz = dmat->bounce_zone; 539 540 /* Initialize the new map */ 541 STAILQ_INIT(&((*mapp)->bpages)); 542 543 /* 544 * Attempt to add pages to our pool on a per-instance 545 * basis up to a sane limit. 546 */ 547 maxpages = MAX_BPAGES; 548 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 549 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 550 int pages; 551 552 pages = MAX(atop(dmat->maxsize), 1); 553 pages = MIN(maxpages - bz->total_bpages, pages); 554 pages = MAX(pages, 1); 555 if (alloc_bounce_pages(dmat, pages) < pages) 556 error = ENOMEM; 557 558 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 559 if (error == 0) 560 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 561 } else { 562 error = 0; 563 } 564 } 565 bz->map_count++; 566 } 567 if (error == 0) 568 dmat->map_count++; 569 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 570 __func__, dmat, dmat->flags, error); 571 return (error); 572} 573 574/* 575 * Destroy a handle for mapping from kva/uva/physical 576 * address space into bus device space. 577 */ 578int 579bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 580{ 581 if (STAILQ_FIRST(&map->bpages) != NULL || 582 STAILQ_FIRST(&map->slist) != NULL) { 583 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 584 __func__, dmat, EBUSY); 585 return (EBUSY); 586 } 587 if (dmat->bounce_zone) 588 dmat->bounce_zone->map_count--; 589 free(map, M_DEVBUF); 590 dmat->map_count--; 591 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 592 return (0); 593} 594 595 596/* 597 * Allocate a piece of memory that can be efficiently mapped into 598 * bus device space based on the constraints lited in the dma tag. 599 * A dmamap to for use with dmamap_load is also allocated. 600 */ 601int 602bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 603 bus_dmamap_t *mapp) 604{ 605 busdma_bufalloc_t ba; 606 struct busdma_bufzone *bufzone; 607 vm_memattr_t memattr; 608 int mflags; 609 610 if (flags & BUS_DMA_NOWAIT) 611 mflags = M_NOWAIT; 612 else 613 mflags = M_WAITOK; 614 615 /* ARM non-snooping caches need a map for the VA cache sync structure */ 616 617 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 618 M_NOWAIT | M_ZERO); 619 if (*mapp == NULL) { 620 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 621 __func__, dmat, dmat->flags, ENOMEM); 622 return (ENOMEM); 623 } 624 625 STAILQ_INIT(&((*mapp)->slist)); 626 627 if (dmat->segments == NULL) { 628 dmat->segments = (bus_dma_segment_t *)malloc( 629 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 630 mflags); 631 if (dmat->segments == NULL) { 632 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 633 __func__, dmat, dmat->flags, ENOMEM); 634 free(*mapp, M_DEVBUF); 635 *mapp = NULL; 636 return (ENOMEM); 637 } 638 } 639 640 if (flags & BUS_DMA_ZERO) 641 mflags |= M_ZERO; 642 if (flags & BUS_DMA_COHERENT) { 643 memattr = VM_MEMATTR_UNCACHEABLE; 644 ba = coherent_allocator; 645 (*mapp)->flags |= DMAMAP_COHERENT; 646 } else { 647 memattr = VM_MEMATTR_DEFAULT; 648 ba = standard_allocator; 649 (*mapp)->flags = 0; 650 } 651#ifdef notyet 652 /* All buffers we allocate are cache-aligned. */ 653 map->flags |= DMAMAP_CACHE_ALIGNED; 654#endif 655 656 /* 657 * Try to find a bufzone in the allocator that holds a cache of buffers 658 * of the right size for this request. If the buffer is too big to be 659 * held in the allocator cache, this returns NULL. 660 */ 661 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 662 663 /* 664 * Allocate the buffer from the uma(9) allocator if... 665 * - It's small enough to be in the allocator (bufzone not NULL). 666 * - The alignment constraint isn't larger than the allocation size 667 * (the allocator aligns buffers to their size boundaries). 668 * - There's no need to handle lowaddr/highaddr exclusion zones. 669 * else allocate non-contiguous pages if... 670 * - The page count that could get allocated doesn't exceed nsegments. 671 * - The alignment constraint isn't larger than a page boundary. 672 * - There are no boundary-crossing constraints. 673 * else allocate a block of contiguous pages because one or more of the 674 * constraints is something that only the contig allocator can fulfill. 675 */ 676 if (bufzone != NULL && dmat->alignment <= bufzone->size && 677 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 678 *vaddr = uma_zalloc(bufzone->umazone, mflags); 679 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 680 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 681 *vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize, 682 mflags, 0, dmat->lowaddr, memattr); 683 } else { 684 *vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize, 685 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 686 memattr); 687 } 688 689 690 if (*vaddr == NULL) { 691 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 692 __func__, dmat, dmat->flags, ENOMEM); 693 free(*mapp, M_DEVBUF); 694 *mapp = NULL; 695 return (ENOMEM); 696 } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { 697 printf("bus_dmamem_alloc failed to align memory properly.\n"); 698 } 699 dmat->map_count++; 700 701 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 702 __func__, dmat, dmat->flags, 0); 703 return (0); 704} 705 706/* 707 * Free a piece of memory and it's allociated dmamap, that was allocated 708 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 709 */ 710void 711bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 712{ 713 struct busdma_bufzone *bufzone; 714 busdma_bufalloc_t ba; 715 716 if (map->flags & DMAMAP_COHERENT) 717 ba = coherent_allocator; 718 else 719 ba = standard_allocator; 720 721 /* Be careful not to access map from here on. */ 722 723 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 724 725 if (bufzone != NULL && dmat->alignment <= bufzone->size && 726 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 727 uma_zfree(bufzone->umazone, vaddr); 728 else 729 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); 730 731 dmat->map_count--; 732 free(map, M_DEVBUF); 733 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 734} 735 736static int 737_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 738 void *buf, bus_size_t buflen, int flags) 739{ 740 vm_offset_t vaddr; 741 vm_offset_t vendaddr; 742 bus_addr_t paddr; 743 744 if (map->pagesneeded == 0) { 745 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 746 " map= %p, pagesneeded= %d", 747 dmat->lowaddr, dmat->boundary, dmat->alignment, 748 map, map->pagesneeded); 749 /* 750 * Count the number of bounce pages 751 * needed in order to complete this transfer 752 */ 753 vaddr = (vm_offset_t)buf; 754 vendaddr = (vm_offset_t)buf + buflen; 755 756 while (vaddr < vendaddr) { 757 if (__predict_true(map->pmap == pmap_kernel())) 758 paddr = pmap_kextract(vaddr); 759 else 760 paddr = pmap_extract(map->pmap, vaddr); 761 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 762 run_filter(dmat, paddr) != 0) { 763 map->pagesneeded++; 764 } 765 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); 766 767 } 768 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 769 } 770 771 /* Reserve Necessary Bounce Pages */ 772 if (map->pagesneeded != 0) { 773 mtx_lock(&bounce_lock); 774 if (flags & BUS_DMA_NOWAIT) { 775 if (reserve_bounce_pages(dmat, map, 0) != 0) { 776 map->pagesneeded = 0; 777 mtx_unlock(&bounce_lock); 778 return (ENOMEM); 779 } 780 } else { 781 if (reserve_bounce_pages(dmat, map, 1) != 0) { 782 /* Queue us for resources */ 783 map->dmat = dmat; 784 map->buf = buf; 785 map->buflen = buflen; 786 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 787 map, links); 788 mtx_unlock(&bounce_lock); 789 return (EINPROGRESS); 790 } 791 } 792 mtx_unlock(&bounce_lock); 793 } 794 795 return (0); 796} 797 798/* 799 * Utility function to load a linear buffer. lastaddrp holds state 800 * between invocations (for multiple-buffer loads). segp contains 801 * the starting segment on entrace, and the ending segment on exit. 802 * first indicates if this is the first invocation of this function. 803 */ 804static __inline int 805_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 806 bus_dmamap_t map, 807 void *buf, bus_size_t buflen, 808 int flags, 809 bus_addr_t *lastaddrp, 810 bus_dma_segment_t *segs, 811 int *segp, 812 int first) 813{ 814 bus_size_t sgsize; 815 bus_addr_t curaddr, lastaddr, baddr, bmask; 816 vm_offset_t vaddr; 817 struct sync_list *sl; 818 int seg, error; 819 820 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 821 error = _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); 822 if (error) 823 return (error); 824 } 825 826 sl = NULL; 827 vaddr = (vm_offset_t)buf; 828 lastaddr = *lastaddrp; 829 bmask = ~(dmat->boundary - 1); 830 831 for (seg = *segp; buflen > 0 ; ) { 832 /* 833 * Get the physical address for this segment. 834 */ 835 if (__predict_true(map->pmap == pmap_kernel())) 836 curaddr = pmap_kextract(vaddr); 837 else 838 curaddr = pmap_extract(map->pmap, vaddr); 839 840 /* 841 * Compute the segment size, and adjust counts. 842 */ 843 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 844 if (sgsize > dmat->maxsegsz) 845 sgsize = dmat->maxsegsz; 846 if (buflen < sgsize) 847 sgsize = buflen; 848 849 /* 850 * Make sure we don't cross any boundaries. 851 */ 852 if (dmat->boundary > 0) { 853 baddr = (curaddr + dmat->boundary) & bmask; 854 if (sgsize > (baddr - curaddr)) 855 sgsize = (baddr - curaddr); 856 } 857 858 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 859 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 860 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 861 } else { 862 /* add_sync_list(dmat, map, vaddr, sgsize, cflag); */ 863 sl = (struct sync_list *)malloc(sizeof(struct sync_list), 864 M_DEVBUF, M_NOWAIT | M_ZERO); 865 if (sl == NULL) 866 goto cleanup; 867 STAILQ_INSERT_TAIL(&(map->slist), sl, slinks); 868 sl->vaddr = vaddr; 869 sl->datacount = sgsize; 870 sl->busaddr = curaddr; 871 } 872 873 874 if (dmat->ranges) { 875 struct arm32_dma_range *dr; 876 877 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 878 curaddr); 879 if (dr == NULL) { 880 _bus_dmamap_unload(dmat, map); 881 return (EINVAL); 882 } 883 /* 884 * In a valid DMA range. Translate the physical 885 * memory address to an address in the DMA window. 886 */ 887 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 888 } 889 890 /* 891 * Insert chunk into a segment, coalescing with 892 * previous segment if possible. 893 */ 894 if (first) { 895 segs[seg].ds_addr = curaddr; 896 segs[seg].ds_len = sgsize; 897 first = 0; 898 } else { 899 if (curaddr == lastaddr && 900 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 901 (dmat->boundary == 0 || 902 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 903 segs[seg].ds_len += sgsize; 904 else { 905 if (++seg >= dmat->nsegments) 906 break; 907 segs[seg].ds_addr = curaddr; 908 segs[seg].ds_len = sgsize; 909 } 910 } 911 912 lastaddr = curaddr + sgsize; 913 vaddr += sgsize; 914 buflen -= sgsize; 915 } 916 917 *segp = seg; 918 *lastaddrp = lastaddr; 919cleanup: 920 /* 921 * Did we fit? 922 */ 923 if (buflen != 0) { 924 _bus_dmamap_unload(dmat, map); 925 return(EFBIG); /* XXX better return value here? */ 926 } 927 return (0); 928} 929 930/* 931 * Map the buffer buf into bus space using the dmamap map. 932 */ 933int 934bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 935 bus_size_t buflen, bus_dmamap_callback_t *callback, 936 void *callback_arg, int flags) 937{ 938 bus_addr_t lastaddr = 0; 939 int error, nsegs = 0; 940 941 flags |= BUS_DMA_WAITOK; 942 map->callback = callback; 943 map->callback_arg = callback_arg; 944 map->pmap = kernel_pmap; 945 946 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, flags, 947 &lastaddr, dmat->segments, &nsegs, 1); 948 949 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 950 __func__, dmat, dmat->flags, error, nsegs + 1); 951 952 if (error == EINPROGRESS) { 953 return (error); 954 } 955 956 if (error) 957 (*callback)(callback_arg, dmat->segments, 0, error); 958 else 959 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 960 961 /* 962 * Return ENOMEM to the caller so that it can pass it up the stack. 963 * This error only happens when NOWAIT is set, so deferal is disabled. 964 */ 965 if (error == ENOMEM) 966 return (error); 967 968 return (0); 969} 970 971 972/* 973 * Like _bus_dmamap_load(), but for mbufs. 974 */ 975static __inline int 976_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 977 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 978 int flags) 979{ 980 int error; 981 982 M_ASSERTPKTHDR(m0); 983 map->pmap = kernel_pmap; 984 985 flags |= BUS_DMA_NOWAIT; 986 *nsegs = 0; 987 error = 0; 988 if (m0->m_pkthdr.len <= dmat->maxsize) { 989 int first = 1; 990 bus_addr_t lastaddr = 0; 991 struct mbuf *m; 992 993 for (m = m0; m != NULL && error == 0; m = m->m_next) { 994 if (m->m_len > 0) { 995 error = _bus_dmamap_load_buffer(dmat, map, 996 m->m_data, m->m_len, 997 flags, &lastaddr, 998 segs, nsegs, first); 999 first = 0; 1000 } 1001 } 1002 } else { 1003 error = EINVAL; 1004 } 1005 1006 /* XXX FIXME: Having to increment nsegs is really annoying */ 1007 ++*nsegs; 1008 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1009 __func__, dmat, dmat->flags, error, *nsegs); 1010 return (error); 1011} 1012 1013int 1014bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 1015 struct mbuf *m0, 1016 bus_dmamap_callback2_t *callback, void *callback_arg, 1017 int flags) 1018{ 1019 int nsegs, error; 1020 1021 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, 1022 flags); 1023 1024 if (error) { 1025 /* force "no valid mappings" in callback */ 1026 (*callback)(callback_arg, dmat->segments, 0, 0, error); 1027 } else { 1028 (*callback)(callback_arg, dmat->segments, 1029 nsegs, m0->m_pkthdr.len, error); 1030 } 1031 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1032 __func__, dmat, dmat->flags, error, nsegs); 1033 1034 return (error); 1035} 1036 1037int 1038bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 1039 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 1040 int flags) 1041{ 1042 return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); 1043} 1044 1045/* 1046 * Like _bus_dmamap_load(), but for uios. 1047 */ 1048int 1049bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 1050 struct uio *uio, 1051 bus_dmamap_callback2_t *callback, void *callback_arg, 1052 int flags) 1053{ 1054 bus_addr_t lastaddr; 1055 int nsegs, error, first, i; 1056 bus_size_t resid; 1057 struct iovec *iov; 1058 1059 flags |= BUS_DMA_NOWAIT; 1060 resid = uio->uio_resid; 1061 iov = uio->uio_iov; 1062 1063 if (uio->uio_segflg == UIO_USERSPACE) { 1064 KASSERT(uio->uio_td != NULL, 1065 ("bus_dmamap_load_uio: USERSPACE but no proc")); 1066 map->pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 1067 } else 1068 map->pmap = kernel_pmap; 1069 1070 nsegs = 0; 1071 error = 0; 1072 first = 1; 1073 lastaddr = (bus_addr_t) 0; 1074 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 1075 /* 1076 * Now at the first iovec to load. Load each iovec 1077 * until we have exhausted the residual count. 1078 */ 1079 bus_size_t minlen = 1080 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1081 caddr_t addr = (caddr_t) iov[i].iov_base; 1082 1083 if (minlen > 0) { 1084 error = _bus_dmamap_load_buffer(dmat, map, 1085 addr, minlen, flags, &lastaddr, 1086 dmat->segments, &nsegs, first); 1087 first = 0; 1088 resid -= minlen; 1089 } 1090 } 1091 1092 if (error) { 1093 /* force "no valid mappings" in callback */ 1094 (*callback)(callback_arg, dmat->segments, 0, 0, error); 1095 } else { 1096 (*callback)(callback_arg, dmat->segments, 1097 nsegs+1, uio->uio_resid, error); 1098 } 1099 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1100 __func__, dmat, dmat->flags, error, nsegs + 1); 1101 return (error); 1102} 1103 1104/* 1105 * Release the mapping held by map. 1106 */ 1107void 1108_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1109{ 1110 struct bounce_page *bpage; 1111 struct bounce_zone *bz; 1112 struct sync_list *sl; 1113 1114 while ((sl = STAILQ_FIRST(&map->slist)) != NULL) { 1115 STAILQ_REMOVE_HEAD(&map->slist, slinks); 1116 free(sl, M_DEVBUF); 1117 } 1118 1119 if ((bz = dmat->bounce_zone) != NULL) { 1120 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1121 STAILQ_REMOVE_HEAD(&map->bpages, links); 1122 free_bounce_page(dmat, bpage); 1123 } 1124 1125 bz = dmat->bounce_zone; 1126 bz->free_bpages += map->pagesreserved; 1127 bz->reserved_bpages -= map->pagesreserved; 1128 map->pagesreserved = 0; 1129 map->pagesneeded = 0; 1130 } 1131} 1132 1133#ifdef notyetbounceuser 1134 /* If busdma uses user pages, then the interrupt handler could 1135 * be use the kernel vm mapping. Both bounce pages and sync list 1136 * do not cross page boundaries. 1137 * Below is a rough sequence that a person would do to fix the 1138 * user page reference in the kernel vmspace. This would be 1139 * done in the dma post routine. 1140 */ 1141void 1142_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len, 1143 pmap_t pmap, int op) 1144{ 1145 bus_size_t sgsize; 1146 bus_addr_t curaddr; 1147 vm_offset_t va; 1148 1149 /* each synclist entry is contained within a single page. 1150 * 1151 * this would be needed if BUS_DMASYNC_POSTxxxx was implemented 1152 */ 1153 curaddr = pmap_extract(pmap, buf); 1154 va = pmap_dma_map(curaddr); 1155 switch (op) { 1156 case SYNC_USER_INV: 1157 cpu_dcache_wb_range(va, sgsize); 1158 break; 1159 1160 case SYNC_USER_COPYTO: 1161 bcopy((void *)va, (void *)bounce, sgsize); 1162 break; 1163 1164 case SYNC_USER_COPYFROM: 1165 bcopy((void *) bounce, (void *)va, sgsize); 1166 break; 1167 1168 default: 1169 break; 1170 } 1171 1172 pmap_dma_unmap(va); 1173} 1174#endif 1175 1176#ifdef ARM_L2_PIPT 1177#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size) 1178#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size) 1179#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size) 1180#else 1181#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size) 1182#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size) 1183#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size) 1184#endif 1185 1186void 1187_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1188{ 1189 struct bounce_page *bpage; 1190 struct sync_list *sl; 1191 bus_size_t len, unalign; 1192 vm_offset_t buf, ebuf; 1193#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD 1194 vm_offset_t bbuf; 1195 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1196#endif 1197 int listcount = 0; 1198 1199 /* if buffer was from user space, it it possible that this 1200 * is not the same vm map. The fix is to map each page in 1201 * the buffer into the current address space (KVM) and then 1202 * do the bounce copy or sync list cache operation. 1203 * 1204 * The sync list entries are already broken into 1205 * their respective physical pages. 1206 */ 1207 if (!pmap_dmap_iscurrent(map->pmap)) 1208 printf("_bus_dmamap_sync: wrong user map: %p %x\n", map->pmap, op); 1209 1210 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1211 1212 /* Handle data bouncing. */ 1213 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1214 "performing bounce", __func__, dmat, dmat->flags, op); 1215 1216 if (op & BUS_DMASYNC_PREWRITE) { 1217 while (bpage != NULL) { 1218 bcopy((void *)bpage->datavaddr, 1219 (void *)bpage->vaddr, 1220 bpage->datacount); 1221 cpu_dcache_wb_range((vm_offset_t)bpage->vaddr, 1222 bpage->datacount); 1223 l2cache_wb_range((vm_offset_t)bpage->vaddr, 1224 (vm_offset_t)bpage->busaddr, 1225 bpage->datacount); 1226 bpage = STAILQ_NEXT(bpage, links); 1227 } 1228 dmat->bounce_zone->total_bounced++; 1229 } 1230 1231 if (op & BUS_DMASYNC_POSTREAD) { 1232 if (!pmap_dmap_iscurrent(map->pmap)) 1233 panic("_bus_dmamap_sync: wrong user map. apply fix"); 1234 1235 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr, 1236 bpage->datacount); 1237 l2cache_inv_range((vm_offset_t)bpage->vaddr, 1238 (vm_offset_t)bpage->busaddr, 1239 bpage->datacount); 1240 while (bpage != NULL) { 1241 vm_offset_t startv; 1242 vm_paddr_t startp; 1243 int len; 1244 1245 startv = bpage->vaddr &~ arm_dcache_align_mask; 1246 startp = bpage->busaddr &~ arm_dcache_align_mask; 1247 len = bpage->datacount; 1248 1249 if (startv != bpage->vaddr) 1250 len += bpage->vaddr & arm_dcache_align_mask; 1251 if (len & arm_dcache_align_mask) 1252 len = (len - 1253 (len & arm_dcache_align_mask)) + 1254 arm_dcache_align; 1255 cpu_dcache_inv_range(startv, len); 1256 l2cache_inv_range(startv, startp, len); 1257 bcopy((void *)bpage->vaddr, 1258 (void *)bpage->datavaddr, 1259 bpage->datacount); 1260 bpage = STAILQ_NEXT(bpage, links); 1261 } 1262 dmat->bounce_zone->total_bounced++; 1263 } 1264 } 1265 if (map->flags & DMAMAP_COHERENT) 1266 return; 1267 1268 sl = STAILQ_FIRST(&map->slist); 1269 while (sl) { 1270 listcount++; 1271 sl = STAILQ_NEXT(sl, slinks); 1272 } 1273 if ((sl = STAILQ_FIRST(&map->slist)) != NULL) { 1274 /* ARM caches are not self-snooping for dma */ 1275 1276 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1277 "performing sync", __func__, dmat, dmat->flags, op); 1278 1279 switch (op) { 1280 case BUS_DMASYNC_PREWRITE: 1281 while (sl != NULL) { 1282 cpu_dcache_wb_range(sl->vaddr, sl->datacount); 1283 l2cache_wb_range(sl->vaddr, sl->busaddr, 1284 sl->datacount); 1285 sl = STAILQ_NEXT(sl, slinks); 1286 } 1287 break; 1288 1289 case BUS_DMASYNC_PREREAD: 1290 while (sl != NULL) { 1291 /* write back the unaligned portions */ 1292 vm_paddr_t physaddr = sl->busaddr, ephysaddr; 1293 buf = sl->vaddr; 1294 len = sl->datacount; 1295 ebuf = buf + len; /* end of buffer */ 1296 ephysaddr = physaddr + len; 1297 unalign = buf & arm_dcache_align_mask; 1298 if (unalign) { 1299 /* wbinv leading fragment */ 1300 buf &= ~arm_dcache_align_mask; 1301 physaddr &= ~arm_dcache_align_mask; 1302 cpu_dcache_wbinv_range(buf, 1303 arm_dcache_align); 1304 l2cache_wbinv_range(buf, physaddr, 1305 arm_dcache_align); 1306 buf += arm_dcache_align; 1307 physaddr += arm_dcache_align; 1308 /* number byte in buffer wbinv */ 1309 unalign = arm_dcache_align - unalign; 1310 if (len > unalign) 1311 len -= unalign; 1312 else 1313 len = 0; 1314 } 1315 unalign = ebuf & arm_dcache_align_mask; 1316 if (ebuf > buf && unalign) { 1317 /* wbinv trailing fragment */ 1318 len -= unalign; 1319 ebuf -= unalign; 1320 ephysaddr -= unalign; 1321 cpu_dcache_wbinv_range(ebuf, 1322 arm_dcache_align); 1323 l2cache_wbinv_range(ebuf, ephysaddr, 1324 arm_dcache_align); 1325 } 1326 if (ebuf > buf) { 1327 cpu_dcache_inv_range(buf, len); 1328 l2cache_inv_range(buf, physaddr, len); 1329 } 1330 sl = STAILQ_NEXT(sl, slinks); 1331 } 1332 break; 1333 1334 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1335 while (sl != NULL) { 1336 cpu_dcache_wbinv_range(sl->vaddr, sl->datacount); 1337 l2cache_wbinv_range(sl->vaddr, 1338 sl->busaddr, sl->datacount); 1339 sl = STAILQ_NEXT(sl, slinks); 1340 } 1341 break; 1342 1343#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD 1344 case BUS_DMASYNC_POSTREAD: 1345 if (!pmap_dmap_iscurrent(map->pmap)) 1346 panic("_bus_dmamap_sync: wrong user map. apply fix"); 1347 while (sl != NULL) { 1348 /* write back the unaligned portions */ 1349 vm_paddr_t physaddr; 1350 buf = sl->vaddr; 1351 len = sl->datacount; 1352 physaddr = sl->busaddr; 1353 bbuf = buf & ~arm_dcache_align_mask; 1354 ebuf = buf + len; 1355 physaddr = physaddr & ~arm_dcache_align_mask; 1356 unalign = buf & arm_dcache_align_mask; 1357 if (unalign) { 1358 memcpy(_tmp_cl, (void *)bbuf, unalign); 1359 len += unalign; /* inv entire cache line */ 1360 } 1361 unalign = ebuf & arm_dcache_align_mask; 1362 if (unalign) { 1363 unalign = arm_dcache_align - unalign; 1364 memcpy(_tmp_clend, (void *)ebuf, unalign); 1365 len += unalign; /* inv entire cache line */ 1366 } 1367 /* inv are cache length aligned */ 1368 cpu_dcache_inv_range(bbuf, len); 1369 l2cache_inv_range(bbuf, physaddr, len); 1370 1371 unalign = (vm_offset_t)buf & arm_dcache_align_mask; 1372 if (unalign) { 1373 memcpy((void *)bbuf, _tmp_cl, unalign); 1374 } 1375 unalign = ebuf & arm_dcache_align_mask; 1376 if (unalign) { 1377 unalign = arm_dcache_align - unalign; 1378 memcpy((void *)ebuf, _tmp_clend, unalign); 1379 } 1380 sl = STAILQ_NEXT(sl, slinks); 1381 } 1382 break; 1383#endif /* FIX_DMAP_BUS_DMASYNC_POSTREAD */ 1384 1385 default: 1386 break; 1387 } 1388 } 1389} 1390 1391static void 1392init_bounce_pages(void *dummy __unused) 1393{ 1394 1395 total_bpages = 0; 1396 STAILQ_INIT(&bounce_zone_list); 1397 STAILQ_INIT(&bounce_map_waitinglist); 1398 STAILQ_INIT(&bounce_map_callbacklist); 1399 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1400} 1401SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1402 1403static struct sysctl_ctx_list * 1404busdma_sysctl_tree(struct bounce_zone *bz) 1405{ 1406 return (&bz->sysctl_tree); 1407} 1408 1409static struct sysctl_oid * 1410busdma_sysctl_tree_top(struct bounce_zone *bz) 1411{ 1412 return (bz->sysctl_tree_top); 1413} 1414 1415static int 1416alloc_bounce_zone(bus_dma_tag_t dmat) 1417{ 1418 struct bounce_zone *bz; 1419 1420 /* Check to see if we already have a suitable zone */ 1421 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1422 if ((dmat->alignment <= bz->alignment) 1423 && (dmat->lowaddr >= bz->lowaddr)) { 1424 dmat->bounce_zone = bz; 1425 return (0); 1426 } 1427 } 1428 1429 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1430 M_NOWAIT | M_ZERO)) == NULL) 1431 return (ENOMEM); 1432 1433 STAILQ_INIT(&bz->bounce_page_list); 1434 bz->free_bpages = 0; 1435 bz->reserved_bpages = 0; 1436 bz->active_bpages = 0; 1437 bz->lowaddr = dmat->lowaddr; 1438 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1439 bz->map_count = 0; 1440 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1441 busdma_zonecount++; 1442 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1443 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1444 dmat->bounce_zone = bz; 1445 1446 sysctl_ctx_init(&bz->sysctl_tree); 1447 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1448 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1449 CTLFLAG_RD, 0, ""); 1450 if (bz->sysctl_tree_top == NULL) { 1451 sysctl_ctx_free(&bz->sysctl_tree); 1452 return (0); /* XXX error code? */ 1453 } 1454 1455 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1456 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1457 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1458 "Total bounce pages"); 1459 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1460 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1461 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1462 "Free bounce pages"); 1463 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1464 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1465 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1466 "Reserved bounce pages"); 1467 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1468 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1469 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1470 "Active bounce pages"); 1471 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1472 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1473 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1474 "Total bounce requests"); 1475 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1476 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1477 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1478 "Total bounce requests that were deferred"); 1479 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1480 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1481 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1482 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1483 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1484 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1485 1486 return (0); 1487} 1488 1489static int 1490alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1491{ 1492 struct bounce_zone *bz; 1493 int count; 1494 1495 bz = dmat->bounce_zone; 1496 count = 0; 1497 while (numpages > 0) { 1498 struct bounce_page *bpage; 1499 1500 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1501 M_NOWAIT | M_ZERO); 1502 1503 if (bpage == NULL) 1504 break; 1505 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1506 M_NOWAIT, 0ul, 1507 bz->lowaddr, 1508 PAGE_SIZE, 1509 0); 1510 if (bpage->vaddr == 0) { 1511 free(bpage, M_DEVBUF); 1512 break; 1513 } 1514 bpage->busaddr = pmap_kextract(bpage->vaddr); 1515 mtx_lock(&bounce_lock); 1516 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1517 total_bpages++; 1518 bz->total_bpages++; 1519 bz->free_bpages++; 1520 mtx_unlock(&bounce_lock); 1521 count++; 1522 numpages--; 1523 } 1524 return (count); 1525} 1526 1527static int 1528reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1529{ 1530 struct bounce_zone *bz; 1531 int pages; 1532 1533 mtx_assert(&bounce_lock, MA_OWNED); 1534 bz = dmat->bounce_zone; 1535 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1536 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1537 return (map->pagesneeded - (map->pagesreserved + pages)); 1538 bz->free_bpages -= pages; 1539 bz->reserved_bpages += pages; 1540 map->pagesreserved += pages; 1541 pages = map->pagesneeded - map->pagesreserved; 1542 1543 return (pages); 1544} 1545 1546static bus_addr_t 1547add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1548 bus_size_t size) 1549{ 1550 struct bounce_zone *bz; 1551 struct bounce_page *bpage; 1552 1553 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1554 KASSERT(map != NULL, 1555 ("add_bounce_page: bad map %p", map)); 1556 1557 bz = dmat->bounce_zone; 1558 if (map->pagesneeded == 0) 1559 panic("add_bounce_page: map doesn't need any pages"); 1560 map->pagesneeded--; 1561 1562 if (map->pagesreserved == 0) 1563 panic("add_bounce_page: map doesn't need any pages"); 1564 map->pagesreserved--; 1565 1566 mtx_lock(&bounce_lock); 1567 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1568 if (bpage == NULL) 1569 panic("add_bounce_page: free page list is empty"); 1570 1571 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1572 bz->reserved_bpages--; 1573 bz->active_bpages++; 1574 mtx_unlock(&bounce_lock); 1575 1576 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1577 /* Page offset needs to be preserved. */ 1578 bpage->vaddr |= vaddr & PAGE_MASK; 1579 bpage->busaddr |= vaddr & PAGE_MASK; 1580 } 1581 bpage->datavaddr = vaddr; 1582 bpage->datacount = size; 1583 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1584 return (bpage->busaddr); 1585} 1586 1587static void 1588free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1589{ 1590 struct bus_dmamap *map; 1591 struct bounce_zone *bz; 1592 1593 bz = dmat->bounce_zone; 1594 bpage->datavaddr = 0; 1595 bpage->datacount = 0; 1596 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1597 /* 1598 * Reset the bounce page to start at offset 0. Other uses 1599 * of this bounce page may need to store a full page of 1600 * data and/or assume it starts on a page boundary. 1601 */ 1602 bpage->vaddr &= ~PAGE_MASK; 1603 bpage->busaddr &= ~PAGE_MASK; 1604 } 1605 1606 mtx_lock(&bounce_lock); 1607 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1608 bz->free_bpages++; 1609 bz->active_bpages--; 1610 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1611 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1612 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1613 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1614 map, links); 1615 busdma_swi_pending = 1; 1616 bz->total_deferred++; 1617 swi_sched(vm_ih, 0); 1618 } 1619 } 1620 mtx_unlock(&bounce_lock); 1621} 1622 1623void 1624busdma_swi(void) 1625{ 1626 bus_dma_tag_t dmat; 1627 struct bus_dmamap *map; 1628 1629 mtx_lock(&bounce_lock); 1630 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1631 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1632 mtx_unlock(&bounce_lock); 1633 dmat = map->dmat; 1634 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1635 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1636 map->callback, map->callback_arg, /*flags*/0); 1637 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1638 mtx_lock(&bounce_lock); 1639 } 1640 mtx_unlock(&bounce_lock); 1641} 1642