busdma_machdep-v4.c revision 244471
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 244471 2012-12-20 00:38:08Z cognet $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/interrupt.h> 62#include <sys/lock.h> 63#include <sys/proc.h> 64#include <sys/mutex.h> 65#include <sys/mbuf.h> 66#include <sys/uio.h> 67#include <sys/ktr.h> 68#include <sys/kernel.h> 69#include <sys/sysctl.h> 70 71#include <vm/uma.h> 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74#include <vm/vm_kern.h> 75#include <vm/vm_page.h> 76#include <vm/vm_map.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86 87struct bounce_zone; 88 89struct bus_dma_tag { 90 bus_dma_tag_t parent; 91 bus_size_t alignment; 92 bus_addr_t boundary; 93 bus_addr_t lowaddr; 94 bus_addr_t highaddr; 95 bus_dma_filter_t *filter; 96 void *filterarg; 97 bus_size_t maxsize; 98 u_int nsegments; 99 bus_size_t maxsegsz; 100 int flags; 101 int ref_count; 102 int map_count; 103 bus_dma_lock_t *lockfunc; 104 void *lockfuncarg; 105 /* 106 * DMA range for this tag. If the page doesn't fall within 107 * one of these ranges, an error is returned. The caller 108 * may then decide what to do with the transfer. If the 109 * range pointer is NULL, it is ignored. 110 */ 111 struct arm32_dma_range *ranges; 112 int _nranges; 113 struct bounce_zone *bounce_zone; 114 /* 115 * Most tags need one or two segments, and can use the local tagsegs 116 * array. For tags with a larger limit, we'll allocate a bigger array 117 * on first use. 118 */ 119 bus_dma_segment_t *segments; 120 bus_dma_segment_t tagsegs[2]; 121}; 122 123struct bounce_page { 124 vm_offset_t vaddr; /* kva of bounce buffer */ 125 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */ 126 bus_addr_t busaddr; /* Physical address */ 127 vm_offset_t datavaddr; /* kva of client data */ 128 bus_size_t datacount; /* client data count */ 129 STAILQ_ENTRY(bounce_page) links; 130}; 131 132int busdma_swi_pending; 133 134struct bounce_zone { 135 STAILQ_ENTRY(bounce_zone) links; 136 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 137 int total_bpages; 138 int free_bpages; 139 int reserved_bpages; 140 int active_bpages; 141 int total_bounced; 142 int total_deferred; 143 int map_count; 144 bus_size_t alignment; 145 bus_addr_t lowaddr; 146 char zoneid[8]; 147 char lowaddrid[20]; 148 struct sysctl_ctx_list sysctl_tree; 149 struct sysctl_oid *sysctl_tree_top; 150}; 151 152static struct mtx bounce_lock; 153static int total_bpages; 154static int busdma_zonecount; 155static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 156 157static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 158SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 159 "Total bounce pages"); 160 161#define DMAMAP_LINEAR 0x1 162#define DMAMAP_MBUF 0x2 163#define DMAMAP_UIO 0x4 164#define DMAMAP_CACHE_ALIGNED 0x10 165#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) 166#define DMAMAP_COHERENT 0x8 167struct bus_dmamap { 168 struct bp_list bpages; 169 int pagesneeded; 170 int pagesreserved; 171 bus_dma_tag_t dmat; 172 int flags; 173 void *buffer; 174 int len; 175 STAILQ_ENTRY(bus_dmamap) links; 176 bus_dmamap_callback_t *callback; 177 void *callback_arg; 178 179}; 180 181static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 182static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 183 184static struct mtx busdma_mtx; 185 186MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 187 188static void init_bounce_pages(void *dummy); 189static int alloc_bounce_zone(bus_dma_tag_t dmat); 190static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 191static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 192 int commit); 193static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 194 vm_offset_t vaddr, bus_size_t size); 195static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 196 197/* Default tag, as most drivers provide no parent tag. */ 198bus_dma_tag_t arm_root_dma_tag; 199 200//---------------------------------------------------------------------------- 201// Begin block of code useful to transplant to other implementations. 202 203static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 204 205static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 206static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 207 208/* 209 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 210 * It'll need platform-specific changes if this code is copied. 211 */ 212static int 213dmamap_ctor(void *mem, int size, void *arg, int flags) 214{ 215 bus_dmamap_t map; 216 bus_dma_tag_t dmat; 217 218 map = (bus_dmamap_t)mem; 219 dmat = (bus_dma_tag_t)arg; 220 221 dmat->map_count++; 222 223 map->dmat = dmat; 224 map->flags = 0; 225 STAILQ_INIT(&map->bpages); 226 227 return (0); 228} 229 230/* 231 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 232 * It may need platform-specific changes if this code is copied . 233 */ 234static void 235dmamap_dtor(void *mem, int size, void *arg) 236{ 237 bus_dmamap_t map; 238 239 map = (bus_dmamap_t)mem; 240 241 map->dmat->map_count--; 242} 243 244static void 245busdma_init(void *dummy) 246{ 247 248 /* Create a cache of maps for bus_dmamap_create(). */ 249 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 250 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 251 252 /* Create a cache of buffers in standard (cacheable) memory. */ 253 standard_allocator = busdma_bufalloc_create("buffer", 254 arm_dcache_align, /* minimum_alignment */ 255 NULL, /* uma_alloc func */ 256 NULL, /* uma_free func */ 257 0); /* uma_zcreate_flags */ 258 259 /* 260 * Create a cache of buffers in uncacheable memory, to implement the 261 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 262 */ 263 coherent_allocator = busdma_bufalloc_create("coherent", 264 arm_dcache_align, /* minimum_alignment */ 265 busdma_bufalloc_alloc_uncacheable, 266 busdma_bufalloc_free_uncacheable, 267 0); /* uma_zcreate_flags */ 268} 269 270/* 271 * This init historically used SI_SUB_VM, but now the init code requires 272 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 273 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 274 * SI_SUB_KMEM and SI_ORDER_THIRD. 275 */ 276SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 277 278// End block of code useful to transplant to other implementations. 279//---------------------------------------------------------------------------- 280 281/* 282 * Return true if a match is made. 283 * 284 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 285 * 286 * If paddr is within the bounds of the dma tag then call the filter callback 287 * to check for a match, if there is no filter callback then assume a match. 288 */ 289static int 290run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 291{ 292 int retval; 293 294 retval = 0; 295 296 do { 297 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 298 || ((paddr & (dmat->alignment - 1)) != 0)) 299 && (dmat->filter == NULL 300 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 301 retval = 1; 302 303 dmat = dmat->parent; 304 } while (retval == 0 && dmat != NULL); 305 return (retval); 306} 307 308/* 309 * This routine checks the exclusion zone constraints from a tag against the 310 * physical RAM available on the machine. If a tag specifies an exclusion zone 311 * but there's no RAM in that zone, then we avoid allocating resources to bounce 312 * a request, and we can use any memory allocator (as opposed to needing 313 * kmem_alloc_contig() just because it can allocate pages in an address range). 314 * 315 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 316 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 317 * possibly have RAM at an address higher than the highest address we can 318 * express, so we take a fast out. 319 */ 320static __inline int 321_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 322{ 323 int i; 324 325 if (lowaddr >= BUS_SPACE_MAXADDR) 326 return (0); 327 328 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 329 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 330 || (lowaddr < phys_avail[i] && 331 highaddr > phys_avail[i])) 332 return (1); 333 } 334 return (0); 335} 336 337static __inline struct arm32_dma_range * 338_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 339 bus_addr_t curaddr) 340{ 341 struct arm32_dma_range *dr; 342 int i; 343 344 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 345 if (curaddr >= dr->dr_sysbase && 346 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 347 return (dr); 348 } 349 350 return (NULL); 351} 352/* 353 * Convenience function for manipulating driver locks from busdma (during 354 * busdma_swi, for example). Drivers that don't provide their own locks 355 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 356 * non-mutex locking scheme don't have to use this at all. 357 */ 358void 359busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 360{ 361 struct mtx *dmtx; 362 363 dmtx = (struct mtx *)arg; 364 switch (op) { 365 case BUS_DMA_LOCK: 366 mtx_lock(dmtx); 367 break; 368 case BUS_DMA_UNLOCK: 369 mtx_unlock(dmtx); 370 break; 371 default: 372 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 373 } 374} 375 376/* 377 * dflt_lock should never get called. It gets put into the dma tag when 378 * lockfunc == NULL, which is only valid if the maps that are associated 379 * with the tag are meant to never be defered. 380 * XXX Should have a way to identify which driver is responsible here. 381 */ 382static void 383dflt_lock(void *arg, bus_dma_lock_op_t op) 384{ 385#ifdef INVARIANTS 386 panic("driver error: busdma dflt_lock called"); 387#else 388 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 389#endif 390} 391 392/* 393 * Allocate a device specific dma_tag. 394 */ 395#define SEG_NB 1024 396 397int 398bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 399 bus_addr_t boundary, bus_addr_t lowaddr, 400 bus_addr_t highaddr, bus_dma_filter_t *filter, 401 void *filterarg, bus_size_t maxsize, int nsegments, 402 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 403 void *lockfuncarg, bus_dma_tag_t *dmat) 404{ 405 bus_dma_tag_t newtag; 406 int error = 0; 407 /* Return a NULL tag on failure */ 408 *dmat = NULL; 409 if (!parent) 410 parent = arm_root_dma_tag; 411 412 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 413 if (newtag == NULL) { 414 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 415 __func__, newtag, 0, error); 416 return (ENOMEM); 417 } 418 419 newtag->parent = parent; 420 newtag->alignment = alignment ? alignment : 1; 421 newtag->boundary = boundary; 422 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 423 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 424 newtag->filter = filter; 425 newtag->filterarg = filterarg; 426 newtag->maxsize = maxsize; 427 newtag->nsegments = nsegments; 428 newtag->maxsegsz = maxsegsz; 429 newtag->flags = flags; 430 newtag->ref_count = 1; /* Count ourself */ 431 newtag->map_count = 0; 432 newtag->ranges = bus_dma_get_range(); 433 newtag->_nranges = bus_dma_get_range_nb(); 434 if (lockfunc != NULL) { 435 newtag->lockfunc = lockfunc; 436 newtag->lockfuncarg = lockfuncarg; 437 } else { 438 newtag->lockfunc = dflt_lock; 439 newtag->lockfuncarg = NULL; 440 } 441 /* 442 * If all the segments we need fit into the local tagsegs array, set the 443 * pointer now. Otherwise NULL the pointer and an array of segments 444 * will be allocated later, on first use. We don't pre-allocate now 445 * because some tags exist just to pass contraints to children in the 446 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 447 * sure don't want to try to allocate an array for that. 448 */ 449 if (newtag->nsegments <= nitems(newtag->tagsegs)) 450 newtag->segments = newtag->tagsegs; 451 else 452 newtag->segments = NULL; 453 /* 454 * Take into account any restrictions imposed by our parent tag 455 */ 456 if (parent != NULL) { 457 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 458 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 459 if (newtag->boundary == 0) 460 newtag->boundary = parent->boundary; 461 else if (parent->boundary != 0) 462 newtag->boundary = MIN(parent->boundary, 463 newtag->boundary); 464 if ((newtag->filter != NULL) || 465 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 466 newtag->flags |= BUS_DMA_COULD_BOUNCE; 467 if (newtag->filter == NULL) { 468 /* 469 * Short circuit looking at our parent directly 470 * since we have encapsulated all of its information 471 */ 472 newtag->filter = parent->filter; 473 newtag->filterarg = parent->filterarg; 474 newtag->parent = parent->parent; 475 } 476 if (newtag->parent != NULL) 477 atomic_add_int(&parent->ref_count, 1); 478 } 479 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 480 || newtag->alignment > 1) 481 newtag->flags |= BUS_DMA_COULD_BOUNCE; 482 483 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 484 (flags & BUS_DMA_ALLOCNOW) != 0) { 485 struct bounce_zone *bz; 486 487 /* Must bounce */ 488 489 if ((error = alloc_bounce_zone(newtag)) != 0) { 490 free(newtag, M_DEVBUF); 491 return (error); 492 } 493 bz = newtag->bounce_zone; 494 495 if (ptoa(bz->total_bpages) < maxsize) { 496 int pages; 497 498 pages = atop(maxsize) - bz->total_bpages; 499 500 /* Add pages to our bounce pool */ 501 if (alloc_bounce_pages(newtag, pages) < pages) 502 error = ENOMEM; 503 } 504 /* Performed initial allocation */ 505 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 506 } else 507 newtag->bounce_zone = NULL; 508 if (error != 0) 509 free(newtag, M_DEVBUF); 510 else 511 *dmat = newtag; 512 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 513 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 514 515 return (error); 516} 517 518int 519bus_dma_tag_destroy(bus_dma_tag_t dmat) 520{ 521#ifdef KTR 522 bus_dma_tag_t dmat_copy = dmat; 523#endif 524 525 if (dmat != NULL) { 526 527 if (dmat->map_count != 0) 528 return (EBUSY); 529 530 while (dmat != NULL) { 531 bus_dma_tag_t parent; 532 533 parent = dmat->parent; 534 atomic_subtract_int(&dmat->ref_count, 1); 535 if (dmat->ref_count == 0) { 536 if (dmat->segments != NULL && 537 dmat->segments != dmat->tagsegs) 538 free(dmat->segments, M_DEVBUF); 539 free(dmat, M_DEVBUF); 540 /* 541 * Last reference count, so 542 * release our reference 543 * count on our parent. 544 */ 545 dmat = parent; 546 } else 547 dmat = NULL; 548 } 549 } 550 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 551 552 return (0); 553} 554 555#include <sys/kdb.h> 556/* 557 * Allocate a handle for mapping from kva/uva/physical 558 * address space into bus device space. 559 */ 560int 561bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 562{ 563 bus_dmamap_t map; 564 int error = 0; 565 566 map = uma_zalloc_arg(dmamap_zone, dmat, M_WAITOK); 567 *mapp = map; 568 569 /* 570 * If the tag's segments haven't been allocated yet we need to do it 571 * now, because we can't sleep for resources at map load time. 572 */ 573 if (dmat->segments == NULL) 574 dmat->segments = malloc(dmat->nsegments * 575 sizeof(*dmat->segments), M_DEVBUF, M_WAITOK); 576 577 /* 578 * Bouncing might be required if the driver asks for an active 579 * exclusion region, a data alignment that is stricter than 1, and/or 580 * an active address boundary. 581 */ 582 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 583 584 /* Must bounce */ 585 struct bounce_zone *bz; 586 int maxpages; 587 588 if (dmat->bounce_zone == NULL) { 589 if ((error = alloc_bounce_zone(dmat)) != 0) { 590 uma_zfree(dmamap_zone, map); 591 *mapp = NULL; 592 return (error); 593 } 594 } 595 bz = dmat->bounce_zone; 596 597 /* Initialize the new map */ 598 STAILQ_INIT(&((*mapp)->bpages)); 599 600 /* 601 * Attempt to add pages to our pool on a per-instance 602 * basis up to a sane limit. 603 */ 604 maxpages = MAX_BPAGES; 605 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 606 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 607 int pages; 608 609 pages = MAX(atop(dmat->maxsize), 1); 610 pages = MIN(maxpages - bz->total_bpages, pages); 611 pages = MAX(pages, 1); 612 if (alloc_bounce_pages(dmat, pages) < pages) 613 error = ENOMEM; 614 615 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 616 if (error == 0) 617 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 618 } else { 619 error = 0; 620 } 621 } 622 bz->map_count++; 623 } 624 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 625 __func__, dmat, dmat->flags, error); 626 627 return (0); 628} 629 630/* 631 * Destroy a handle for mapping from kva/uva/physical 632 * address space into bus device space. 633 */ 634int 635bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 636{ 637 638 if (STAILQ_FIRST(&map->bpages) != NULL) { 639 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 640 __func__, dmat, EBUSY); 641 return (EBUSY); 642 } 643 uma_zfree(dmamap_zone, map); 644 if (dmat->bounce_zone) 645 dmat->bounce_zone->map_count--; 646 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 647 return (0); 648} 649 650/* 651 * Allocate a piece of memory that can be efficiently mapped into bus device 652 * space based on the constraints listed in the dma tag. Returns a pointer to 653 * the allocated memory, and a pointer to an associated bus_dmamap. 654 */ 655int 656bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 657 bus_dmamap_t *mapp) 658{ 659 void * vaddr; 660 struct busdma_bufzone *bufzone; 661 busdma_bufalloc_t ba; 662 bus_dmamap_t map; 663 int mflags; 664 vm_memattr_t memattr; 665 666 if (flags & BUS_DMA_NOWAIT) 667 mflags = M_NOWAIT; 668 else 669 mflags = M_WAITOK; 670 671 /* 672 * If the tag's segments haven't been allocated yet we need to do it 673 * now, because we can't sleep for resources at map load time. 674 */ 675 if (dmat->segments == NULL) 676 dmat->segments = malloc(dmat->nsegments * 677 sizeof(*dmat->segments), M_DEVBUF, mflags); 678 679 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 680 if (map == NULL) 681 return (ENOMEM); 682 683 if (flags & BUS_DMA_COHERENT) { 684 memattr = VM_MEMATTR_UNCACHEABLE; 685 ba = coherent_allocator; 686 map->flags |= DMAMAP_COHERENT; 687 } else { 688 memattr = VM_MEMATTR_DEFAULT; 689 ba = standard_allocator; 690 } 691 /* All buffers we allocate are cache-aligned. */ 692 map->flags |= DMAMAP_CACHE_ALIGNED; 693 694 if (flags & BUS_DMA_ZERO) 695 mflags |= M_ZERO; 696 697 /* 698 * Try to find a bufzone in the allocator that holds a cache of buffers 699 * of the right size for this request. If the buffer is too big to be 700 * held in the allocator cache, this returns NULL. 701 */ 702 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 703 704 /* 705 * Allocate the buffer from the uma(9) allocator if... 706 * - It's small enough to be in the allocator (bufzone not NULL). 707 * - The alignment constraint isn't larger than the allocation size 708 * (the allocator aligns buffers to their size boundaries). 709 * - There's no need to handle lowaddr/highaddr exclusion zones. 710 * else allocate non-contiguous pages if... 711 * - The page count that could get allocated doesn't exceed nsegments. 712 * - The alignment constraint isn't larger than a page boundary. 713 * - There are no boundary-crossing constraints. 714 * else allocate a block of contiguous pages because one or more of the 715 * constraints is something that only the contig allocator can fulfill. 716 */ 717 if (bufzone != NULL && dmat->alignment <= bufzone->size && 718 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 719 vaddr = uma_zalloc(bufzone->umazone, mflags); 720 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 721 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 722 vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize, 723 mflags, 0, dmat->lowaddr, memattr); 724 } else { 725 vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize, 726 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 727 memattr); 728 } 729 730 if (vaddr == NULL) { 731 uma_zfree(dmamap_zone, map); 732 map = NULL; 733 } 734 735 *vaddrp = vaddr; 736 *mapp = map; 737 738 return (vaddr == NULL ? ENOMEM : 0); 739} 740 741/* 742 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 743 * its associated map. 744 */ 745void 746bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 747{ 748 struct busdma_bufzone *bufzone; 749 busdma_bufalloc_t ba; 750 751 if (map->flags & DMAMAP_COHERENT) 752 ba = coherent_allocator; 753 else 754 ba = standard_allocator; 755 uma_zfree(dmamap_zone, map); 756 757 /* Be careful not to access map from here on. */ 758 759 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 760 761 if (bufzone != NULL && dmat->alignment <= bufzone->size && 762 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 763 uma_zfree(bufzone->umazone, vaddr); 764 else 765 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); 766} 767 768static int 769_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 770 void *buf, bus_size_t buflen, int flags) 771{ 772 vm_offset_t vaddr; 773 vm_offset_t vendaddr; 774 bus_addr_t paddr; 775 776 if ((map->pagesneeded == 0)) { 777 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 778 dmat->lowaddr, dmat->boundary, dmat->alignment); 779 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 780 map, map->pagesneeded); 781 /* 782 * Count the number of bounce pages 783 * needed in order to complete this transfer 784 */ 785 vaddr = trunc_page((vm_offset_t)buf); 786 vendaddr = (vm_offset_t)buf + buflen; 787 788 while (vaddr < vendaddr) { 789 if (__predict_true(pmap == pmap_kernel())) 790 paddr = pmap_kextract(vaddr); 791 else 792 paddr = pmap_extract(pmap, vaddr); 793 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 794 run_filter(dmat, paddr) != 0) 795 map->pagesneeded++; 796 vaddr += PAGE_SIZE; 797 } 798 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 799 } 800 801 /* Reserve Necessary Bounce Pages */ 802 if (map->pagesneeded != 0) { 803 mtx_lock(&bounce_lock); 804 if (flags & BUS_DMA_NOWAIT) { 805 if (reserve_bounce_pages(dmat, map, 0) != 0) { 806 mtx_unlock(&bounce_lock); 807 return (ENOMEM); 808 } 809 } else { 810 if (reserve_bounce_pages(dmat, map, 1) != 0) { 811 /* Queue us for resources */ 812 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 813 map, links); 814 mtx_unlock(&bounce_lock); 815 return (EINPROGRESS); 816 } 817 } 818 mtx_unlock(&bounce_lock); 819 } 820 821 return (0); 822} 823 824/* 825 * Utility function to load a linear buffer. lastaddrp holds state 826 * between invocations (for multiple-buffer loads). segp contains 827 * the starting segment on entrance, and the ending segment on exit. 828 * first indicates if this is the first invocation of this function. 829 */ 830static __inline int 831bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 832 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 833 int flags, vm_offset_t *lastaddrp, int *segp) 834{ 835 bus_size_t sgsize; 836 bus_addr_t curaddr, lastaddr, baddr, bmask; 837 vm_offset_t vaddr = (vm_offset_t)buf; 838 int seg; 839 int error = 0; 840 pd_entry_t *pde; 841 pt_entry_t pte; 842 pt_entry_t *ptep; 843 844 lastaddr = *lastaddrp; 845 bmask = ~(dmat->boundary - 1); 846 847 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 848 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, 849 flags); 850 if (error) 851 return (error); 852 } 853 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 854 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 855 856 for (seg = *segp; buflen > 0 ; ) { 857 /* 858 * Get the physical address for this segment. 859 * 860 * XXX Don't support checking for coherent mappings 861 * XXX in user address space. 862 */ 863 if (__predict_true(pmap == pmap_kernel())) { 864 if (pmap_get_pde_pte(pmap, vaddr, &pde, &ptep) == FALSE) 865 return (EFAULT); 866 867 if (__predict_false(pmap_pde_section(pde))) { 868 if (*pde & L1_S_SUPERSEC) 869 curaddr = (*pde & L1_SUP_FRAME) | 870 (vaddr & L1_SUP_OFFSET); 871 else 872 curaddr = (*pde & L1_S_FRAME) | 873 (vaddr & L1_S_OFFSET); 874 } else { 875 pte = *ptep; 876 KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV, 877 ("INV type")); 878 if (__predict_false((pte & L2_TYPE_MASK) 879 == L2_TYPE_L)) { 880 curaddr = (pte & L2_L_FRAME) | 881 (vaddr & L2_L_OFFSET); 882 } else { 883 curaddr = (pte & L2_S_FRAME) | 884 (vaddr & L2_S_OFFSET); 885 } 886 } 887 } else { 888 curaddr = pmap_extract(pmap, vaddr); 889 map->flags &= ~DMAMAP_COHERENT; 890 } 891 892 /* 893 * Compute the segment size, and adjust counts. 894 */ 895 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 896 if (sgsize > dmat->maxsegsz) 897 sgsize = dmat->maxsegsz; 898 if (buflen < sgsize) 899 sgsize = buflen; 900 901 /* 902 * Make sure we don't cross any boundaries. 903 */ 904 if (dmat->boundary > 0) { 905 baddr = (curaddr + dmat->boundary) & bmask; 906 if (sgsize > (baddr - curaddr)) 907 sgsize = (baddr - curaddr); 908 } 909 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 910 map->pagesneeded != 0 && run_filter(dmat, curaddr)) 911 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 912 913 if (dmat->ranges) { 914 struct arm32_dma_range *dr; 915 916 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 917 curaddr); 918 if (dr == NULL) 919 return (EINVAL); 920 /* 921 * In a valid DMA range. Translate the physical 922 * memory address to an address in the DMA window. 923 */ 924 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 925 926 } 927 928 /* 929 * Insert chunk into a segment, coalescing with 930 * the previous segment if possible. 931 */ 932 if (seg >= 0 && curaddr == lastaddr && 933 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 934 (dmat->boundary == 0 || 935 (segs[seg].ds_addr & bmask) == 936 (curaddr & bmask))) { 937 segs[seg].ds_len += sgsize; 938 goto segdone; 939 } else { 940 if (++seg >= dmat->nsegments) 941 break; 942 segs[seg].ds_addr = curaddr; 943 segs[seg].ds_len = sgsize; 944 } 945 if (error) 946 break; 947segdone: 948 lastaddr = curaddr + sgsize; 949 vaddr += sgsize; 950 buflen -= sgsize; 951 } 952 953 *segp = seg; 954 *lastaddrp = lastaddr; 955 956 /* 957 * Did we fit? 958 */ 959 if (buflen != 0) 960 error = EFBIG; /* XXX better return value here? */ 961 return (error); 962} 963 964/* 965 * Map the buffer buf into bus space using the dmamap map. 966 */ 967int 968bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 969 bus_size_t buflen, bus_dmamap_callback_t *callback, 970 void *callback_arg, int flags) 971{ 972 vm_offset_t lastaddr = 0; 973 int error, nsegs = -1; 974 975 KASSERT(dmat != NULL, ("dmatag is NULL")); 976 KASSERT(map != NULL, ("dmamap is NULL")); 977 map->callback = callback; 978 map->callback_arg = callback_arg; 979 map->flags &= ~DMAMAP_TYPE_MASK; 980 map->flags |= DMAMAP_LINEAR; 981 map->buffer = buf; 982 map->len = buflen; 983 error = bus_dmamap_load_buffer(dmat, 984 dmat->segments, map, buf, buflen, kernel_pmap, 985 flags, &lastaddr, &nsegs); 986 if (error == EINPROGRESS) 987 return (error); 988 if (error) 989 (*callback)(callback_arg, NULL, 0, error); 990 else 991 (*callback)(callback_arg, dmat->segments, nsegs + 1, error); 992 993 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 994 __func__, dmat, dmat->flags, nsegs + 1, error); 995 996 return (error); 997} 998 999/* 1000 * Like bus_dmamap_load(), but for mbufs. 1001 * 1002 * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs. 1003 * 1004 * We know that the way the system allocates and uses mbufs implies that we can 1005 * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line 1006 * flushes. Even though the flush may reference the data area within the mbuf 1007 * that isn't aligned to a cache line, we know the overall mbuf itself is 1008 * properly aligned, and we know that the CPU will not touch the header fields 1009 * before the data area while the DMA is in progress. 1010 */ 1011int 1012bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 1013 bus_dmamap_callback2_t *callback, void *callback_arg, 1014 int flags) 1015{ 1016 int nsegs = -1, error = 0; 1017 1018 M_ASSERTPKTHDR(m0); 1019 1020 flags |= BUS_DMA_NOWAIT; 1021 map->flags &= ~DMAMAP_TYPE_MASK; 1022 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1023 map->buffer = m0; 1024 map->len = 0; 1025 if (m0->m_pkthdr.len <= dmat->maxsize) { 1026 vm_offset_t lastaddr = 0; 1027 struct mbuf *m; 1028 1029 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1030 if (m->m_len > 0) { 1031 error = bus_dmamap_load_buffer(dmat, 1032 dmat->segments, map, m->m_data, m->m_len, 1033 pmap_kernel(), flags, &lastaddr, &nsegs); 1034 map->len += m->m_len; 1035 } 1036 } 1037 } else { 1038 error = EINVAL; 1039 } 1040 1041 if (error) { 1042 /* 1043 * force "no valid mappings" on error in callback. 1044 */ 1045 (*callback)(callback_arg, NULL, 0, 0, error); 1046 } else { 1047 (*callback)(callback_arg, dmat->segments, nsegs + 1, 1048 m0->m_pkthdr.len, error); 1049 } 1050 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1051 __func__, dmat, dmat->flags, error, nsegs + 1); 1052 1053 return (error); 1054} 1055 1056int 1057bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 1058 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 1059 int flags) 1060{ 1061 int error = 0; 1062 M_ASSERTPKTHDR(m0); 1063 1064 flags |= BUS_DMA_NOWAIT; 1065 *nsegs = -1; 1066 map->flags &= ~DMAMAP_TYPE_MASK; 1067 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1068 map->buffer = m0; 1069 map->len = 0; 1070 if (m0->m_pkthdr.len <= dmat->maxsize) { 1071 vm_offset_t lastaddr = 0; 1072 struct mbuf *m; 1073 1074 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1075 if (m->m_len > 0) { 1076 error = bus_dmamap_load_buffer(dmat, segs, map, 1077 m->m_data, m->m_len, 1078 pmap_kernel(), flags, &lastaddr, 1079 nsegs); 1080 map->len += m->m_len; 1081 } 1082 } 1083 } else { 1084 error = EINVAL; 1085 } 1086 1087 /* XXX FIXME: Having to increment nsegs is really annoying */ 1088 ++*nsegs; 1089 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1090 __func__, dmat, dmat->flags, error, *nsegs); 1091 return (error); 1092} 1093 1094/* 1095 * Like bus_dmamap_load(), but for uios. 1096 */ 1097int 1098bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 1099 bus_dmamap_callback2_t *callback, void *callback_arg, 1100 int flags) 1101{ 1102 vm_offset_t lastaddr = 0; 1103 int nsegs, i, error; 1104 bus_size_t resid; 1105 struct iovec *iov; 1106 struct pmap *pmap; 1107 1108 resid = uio->uio_resid; 1109 iov = uio->uio_iov; 1110 map->flags &= ~DMAMAP_TYPE_MASK; 1111 map->flags |= DMAMAP_UIO; 1112 map->buffer = uio; 1113 map->len = 0; 1114 1115 if (uio->uio_segflg == UIO_USERSPACE) { 1116 KASSERT(uio->uio_td != NULL, 1117 ("bus_dmamap_load_uio: USERSPACE but no proc")); 1118 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 1119 } else 1120 pmap = kernel_pmap; 1121 1122 error = 0; 1123 nsegs = -1; 1124 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 1125 /* 1126 * Now at the first iovec to load. Load each iovec 1127 * until we have exhausted the residual count. 1128 */ 1129 bus_size_t minlen = 1130 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1131 caddr_t addr = (caddr_t) iov[i].iov_base; 1132 1133 if (minlen > 0) { 1134 error = bus_dmamap_load_buffer(dmat, dmat->segments, 1135 map, addr, minlen, pmap, flags, &lastaddr, &nsegs); 1136 1137 map->len += minlen; 1138 resid -= minlen; 1139 } 1140 } 1141 1142 if (error) { 1143 /* 1144 * force "no valid mappings" on error in callback. 1145 */ 1146 (*callback)(callback_arg, NULL, 0, 0, error); 1147 } else { 1148 (*callback)(callback_arg, dmat->segments, nsegs+1, 1149 uio->uio_resid, error); 1150 } 1151 1152 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1153 __func__, dmat, dmat->flags, error, nsegs + 1); 1154 return (error); 1155} 1156 1157/* 1158 * Release the mapping held by map. 1159 */ 1160void 1161_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1162{ 1163 struct bounce_page *bpage; 1164 1165 map->flags &= ~DMAMAP_TYPE_MASK; 1166 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1167 STAILQ_REMOVE_HEAD(&map->bpages, links); 1168 free_bounce_page(dmat, bpage); 1169 } 1170 return; 1171} 1172 1173static void 1174bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned) 1175{ 1176 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1177 register_t s; 1178 int partial; 1179 1180 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1181 cpu_dcache_wb_range((vm_offset_t)buf, len); 1182 cpu_l2cache_wb_range((vm_offset_t)buf, len); 1183 } 1184 1185 /* 1186 * If the caller promises the buffer is properly aligned to a cache line 1187 * (even if the call parms make it look like it isn't) we can avoid 1188 * attempting to preserve the non-DMA part of the cache line in the 1189 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1190 * 1191 * This covers the case of mbufs, where we know how they're aligned and 1192 * know the CPU doesn't touch the header in front of the DMA data area 1193 * during the IO, but it may have touched it right before invoking the 1194 * sync, so a PREREAD writeback is required. 1195 * 1196 * It also handles buffers we created in bus_dmamem_alloc(), which are 1197 * always aligned and padded to cache line size even if the IO length 1198 * isn't a multiple of cache line size. In this case the PREREAD 1199 * writeback probably isn't required, but it's harmless. 1200 */ 1201 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1202 1203 if (op & BUS_DMASYNC_PREREAD) { 1204 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1205 cpu_dcache_inv_range((vm_offset_t)buf, len); 1206 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1207 } else { 1208 cpu_dcache_wbinv_range((vm_offset_t)buf, len); 1209 cpu_l2cache_wbinv_range((vm_offset_t)buf, len); 1210 } 1211 } 1212 if (op & BUS_DMASYNC_POSTREAD) { 1213 if (partial && !bufaligned) { 1214 s = intr_disable(); 1215 if ((vm_offset_t)buf & arm_dcache_align_mask) 1216 memcpy(_tmp_cl, (void *)((vm_offset_t)buf & 1217 ~arm_dcache_align_mask), 1218 (vm_offset_t)buf & arm_dcache_align_mask); 1219 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1220 memcpy(_tmp_clend, 1221 (void *)((vm_offset_t)buf + len), 1222 arm_dcache_align - (((vm_offset_t)(buf) + 1223 len) & arm_dcache_align_mask)); 1224 } 1225 cpu_dcache_inv_range((vm_offset_t)buf, len); 1226 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1227 if (partial && !bufaligned) { 1228 if ((vm_offset_t)buf & arm_dcache_align_mask) 1229 memcpy((void *)((vm_offset_t)buf & 1230 ~arm_dcache_align_mask), _tmp_cl, 1231 (vm_offset_t)buf & arm_dcache_align_mask); 1232 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1233 memcpy((void *)((vm_offset_t)buf + len), 1234 _tmp_clend, arm_dcache_align - 1235 (((vm_offset_t)(buf) + len) & 1236 arm_dcache_align_mask)); 1237 intr_restore(s); 1238 } 1239 } 1240} 1241 1242static void 1243_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1244{ 1245 struct bounce_page *bpage; 1246 1247 STAILQ_FOREACH(bpage, &map->bpages, links) { 1248 if (op & BUS_DMASYNC_PREWRITE) { 1249 bcopy((void *)bpage->datavaddr, 1250 (void *)(bpage->vaddr_nocache != 0 ? 1251 bpage->vaddr_nocache : bpage->vaddr), 1252 bpage->datacount); 1253 if (bpage->vaddr_nocache == 0) { 1254 cpu_dcache_wb_range(bpage->vaddr, 1255 bpage->datacount); 1256 cpu_l2cache_wb_range(bpage->vaddr, 1257 bpage->datacount); 1258 } 1259 dmat->bounce_zone->total_bounced++; 1260 } 1261 if (op & BUS_DMASYNC_POSTREAD) { 1262 if (bpage->vaddr_nocache == 0) { 1263 cpu_dcache_inv_range(bpage->vaddr, 1264 bpage->datacount); 1265 cpu_l2cache_inv_range(bpage->vaddr, 1266 bpage->datacount); 1267 } 1268 bcopy((void *)(bpage->vaddr_nocache != 0 ? 1269 bpage->vaddr_nocache : bpage->vaddr), 1270 (void *)bpage->datavaddr, bpage->datacount); 1271 dmat->bounce_zone->total_bounced++; 1272 } 1273 } 1274} 1275 1276static __inline int 1277_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len) 1278{ 1279 struct bounce_page *bpage; 1280 1281 STAILQ_FOREACH(bpage, &map->bpages, links) { 1282 if ((vm_offset_t)buf >= bpage->datavaddr && 1283 (vm_offset_t)buf + len <= bpage->datavaddr + 1284 bpage->datacount) 1285 return (1); 1286 } 1287 return (0); 1288 1289} 1290 1291void 1292_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1293{ 1294 struct mbuf *m; 1295 struct uio *uio; 1296 int resid; 1297 struct iovec *iov; 1298 int bufaligned; 1299 1300 if (op == BUS_DMASYNC_POSTWRITE) 1301 return; 1302 if (map->flags & DMAMAP_COHERENT) 1303 goto drain; 1304 if (STAILQ_FIRST(&map->bpages)) 1305 _bus_dmamap_sync_bp(dmat, map, op); 1306 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1307 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1308 switch(map->flags & DMAMAP_TYPE_MASK) { 1309 case DMAMAP_LINEAR: 1310 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) 1311 bus_dmamap_sync_buf(map->buffer, map->len, op, 1312 bufaligned); 1313 break; 1314 case DMAMAP_MBUF: 1315 m = map->buffer; 1316 while (m) { 1317 if (m->m_len > 0 && 1318 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) 1319 bus_dmamap_sync_buf(m->m_data, m->m_len, op, 1320 bufaligned); 1321 m = m->m_next; 1322 } 1323 break; 1324 case DMAMAP_UIO: 1325 uio = map->buffer; 1326 iov = uio->uio_iov; 1327 resid = uio->uio_resid; 1328 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1329 bus_size_t minlen = resid < iov[i].iov_len ? resid : 1330 iov[i].iov_len; 1331 if (minlen > 0) { 1332 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, 1333 minlen)) 1334 bus_dmamap_sync_buf(iov[i].iov_base, 1335 minlen, op, bufaligned); 1336 resid -= minlen; 1337 } 1338 } 1339 break; 1340 default: 1341 break; 1342 } 1343 1344drain: 1345 1346 cpu_drain_writebuf(); 1347} 1348 1349static void 1350init_bounce_pages(void *dummy __unused) 1351{ 1352 1353 total_bpages = 0; 1354 STAILQ_INIT(&bounce_zone_list); 1355 STAILQ_INIT(&bounce_map_waitinglist); 1356 STAILQ_INIT(&bounce_map_callbacklist); 1357 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1358} 1359SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1360 1361static struct sysctl_ctx_list * 1362busdma_sysctl_tree(struct bounce_zone *bz) 1363{ 1364 return (&bz->sysctl_tree); 1365} 1366 1367static struct sysctl_oid * 1368busdma_sysctl_tree_top(struct bounce_zone *bz) 1369{ 1370 return (bz->sysctl_tree_top); 1371} 1372 1373static int 1374alloc_bounce_zone(bus_dma_tag_t dmat) 1375{ 1376 struct bounce_zone *bz; 1377 1378 /* Check to see if we already have a suitable zone */ 1379 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1380 if ((dmat->alignment <= bz->alignment) 1381 && (dmat->lowaddr >= bz->lowaddr)) { 1382 dmat->bounce_zone = bz; 1383 return (0); 1384 } 1385 } 1386 1387 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1388 M_NOWAIT | M_ZERO)) == NULL) 1389 return (ENOMEM); 1390 1391 STAILQ_INIT(&bz->bounce_page_list); 1392 bz->free_bpages = 0; 1393 bz->reserved_bpages = 0; 1394 bz->active_bpages = 0; 1395 bz->lowaddr = dmat->lowaddr; 1396 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1397 bz->map_count = 0; 1398 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1399 busdma_zonecount++; 1400 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1401 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1402 dmat->bounce_zone = bz; 1403 1404 sysctl_ctx_init(&bz->sysctl_tree); 1405 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1406 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1407 CTLFLAG_RD, 0, ""); 1408 if (bz->sysctl_tree_top == NULL) { 1409 sysctl_ctx_free(&bz->sysctl_tree); 1410 return (0); /* XXX error code? */ 1411 } 1412 1413 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1414 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1415 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1416 "Total bounce pages"); 1417 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1418 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1419 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1420 "Free bounce pages"); 1421 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1422 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1423 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1424 "Reserved bounce pages"); 1425 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1426 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1427 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1428 "Active bounce pages"); 1429 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1430 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1431 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1432 "Total bounce requests"); 1433 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1434 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1435 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1436 "Total bounce requests that were deferred"); 1437 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1438 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1439 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1440 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1441 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1442 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1443 1444 return (0); 1445} 1446 1447static int 1448alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1449{ 1450 struct bounce_zone *bz; 1451 int count; 1452 1453 bz = dmat->bounce_zone; 1454 count = 0; 1455 while (numpages > 0) { 1456 struct bounce_page *bpage; 1457 1458 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1459 M_NOWAIT | M_ZERO); 1460 1461 if (bpage == NULL) 1462 break; 1463 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1464 M_NOWAIT, 0ul, 1465 bz->lowaddr, 1466 PAGE_SIZE, 1467 0); 1468 if (bpage->vaddr == 0) { 1469 free(bpage, M_DEVBUF); 1470 break; 1471 } 1472 bpage->busaddr = pmap_kextract(bpage->vaddr); 1473 bpage->vaddr_nocache = (vm_offset_t)arm_remap_nocache( 1474 (void *)bpage->vaddr, PAGE_SIZE); 1475 mtx_lock(&bounce_lock); 1476 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1477 total_bpages++; 1478 bz->total_bpages++; 1479 bz->free_bpages++; 1480 mtx_unlock(&bounce_lock); 1481 count++; 1482 numpages--; 1483 } 1484 return (count); 1485} 1486 1487static int 1488reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1489{ 1490 struct bounce_zone *bz; 1491 int pages; 1492 1493 mtx_assert(&bounce_lock, MA_OWNED); 1494 bz = dmat->bounce_zone; 1495 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1496 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1497 return (map->pagesneeded - (map->pagesreserved + pages)); 1498 bz->free_bpages -= pages; 1499 bz->reserved_bpages += pages; 1500 map->pagesreserved += pages; 1501 pages = map->pagesneeded - map->pagesreserved; 1502 1503 return (pages); 1504} 1505 1506static bus_addr_t 1507add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1508 bus_size_t size) 1509{ 1510 struct bounce_zone *bz; 1511 struct bounce_page *bpage; 1512 1513 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1514 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1515 1516 bz = dmat->bounce_zone; 1517 if (map->pagesneeded == 0) 1518 panic("add_bounce_page: map doesn't need any pages"); 1519 map->pagesneeded--; 1520 1521 if (map->pagesreserved == 0) 1522 panic("add_bounce_page: map doesn't need any pages"); 1523 map->pagesreserved--; 1524 1525 mtx_lock(&bounce_lock); 1526 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1527 if (bpage == NULL) 1528 panic("add_bounce_page: free page list is empty"); 1529 1530 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1531 bz->reserved_bpages--; 1532 bz->active_bpages++; 1533 mtx_unlock(&bounce_lock); 1534 1535 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1536 /* Page offset needs to be preserved. */ 1537 bpage->vaddr |= vaddr & PAGE_MASK; 1538 bpage->busaddr |= vaddr & PAGE_MASK; 1539 } 1540 bpage->datavaddr = vaddr; 1541 bpage->datacount = size; 1542 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1543 return (bpage->busaddr); 1544} 1545 1546static void 1547free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1548{ 1549 struct bus_dmamap *map; 1550 struct bounce_zone *bz; 1551 1552 bz = dmat->bounce_zone; 1553 bpage->datavaddr = 0; 1554 bpage->datacount = 0; 1555 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1556 /* 1557 * Reset the bounce page to start at offset 0. Other uses 1558 * of this bounce page may need to store a full page of 1559 * data and/or assume it starts on a page boundary. 1560 */ 1561 bpage->vaddr &= ~PAGE_MASK; 1562 bpage->busaddr &= ~PAGE_MASK; 1563 } 1564 1565 mtx_lock(&bounce_lock); 1566 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1567 bz->free_bpages++; 1568 bz->active_bpages--; 1569 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1570 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1571 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1572 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1573 map, links); 1574 busdma_swi_pending = 1; 1575 bz->total_deferred++; 1576 swi_sched(vm_ih, 0); 1577 } 1578 } 1579 mtx_unlock(&bounce_lock); 1580} 1581 1582void 1583busdma_swi(void) 1584{ 1585 bus_dma_tag_t dmat; 1586 struct bus_dmamap *map; 1587 1588 mtx_lock(&bounce_lock); 1589 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1590 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1591 mtx_unlock(&bounce_lock); 1592 dmat = map->dmat; 1593 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1594 bus_dmamap_load(map->dmat, map, map->buffer, map->len, 1595 map->callback, map->callback_arg, /*flags*/0); 1596 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1597 mtx_lock(&bounce_lock); 1598 } 1599 mtx_unlock(&bounce_lock); 1600} 1601