busdma_machdep-v4.c revision 246158
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 246158 2013-01-31 20:53:31Z kib $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/interrupt.h> 62#include <sys/lock.h> 63#include <sys/proc.h> 64#include <sys/mutex.h> 65#include <sys/mbuf.h> 66#include <sys/uio.h> 67#include <sys/ktr.h> 68#include <sys/kernel.h> 69#include <sys/sysctl.h> 70 71#include <vm/uma.h> 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74#include <vm/vm_kern.h> 75#include <vm/vm_page.h> 76#include <vm/vm_map.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86 87struct bounce_zone; 88 89struct bus_dma_tag { 90 bus_dma_tag_t parent; 91 bus_size_t alignment; 92 bus_addr_t boundary; 93 bus_addr_t lowaddr; 94 bus_addr_t highaddr; 95 bus_dma_filter_t *filter; 96 void *filterarg; 97 bus_size_t maxsize; 98 u_int nsegments; 99 bus_size_t maxsegsz; 100 int flags; 101 int ref_count; 102 int map_count; 103 bus_dma_lock_t *lockfunc; 104 void *lockfuncarg; 105 /* 106 * DMA range for this tag. If the page doesn't fall within 107 * one of these ranges, an error is returned. The caller 108 * may then decide what to do with the transfer. If the 109 * range pointer is NULL, it is ignored. 110 */ 111 struct arm32_dma_range *ranges; 112 int _nranges; 113 struct bounce_zone *bounce_zone; 114 /* 115 * Most tags need one or two segments, and can use the local tagsegs 116 * array. For tags with a larger limit, we'll allocate a bigger array 117 * on first use. 118 */ 119 bus_dma_segment_t *segments; 120 bus_dma_segment_t tagsegs[2]; 121}; 122 123struct bounce_page { 124 vm_offset_t vaddr; /* kva of bounce buffer */ 125 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */ 126 bus_addr_t busaddr; /* Physical address */ 127 vm_offset_t datavaddr; /* kva of client data */ 128 bus_size_t datacount; /* client data count */ 129 STAILQ_ENTRY(bounce_page) links; 130}; 131 132int busdma_swi_pending; 133 134struct bounce_zone { 135 STAILQ_ENTRY(bounce_zone) links; 136 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 137 int total_bpages; 138 int free_bpages; 139 int reserved_bpages; 140 int active_bpages; 141 int total_bounced; 142 int total_deferred; 143 int map_count; 144 bus_size_t alignment; 145 bus_addr_t lowaddr; 146 char zoneid[8]; 147 char lowaddrid[20]; 148 struct sysctl_ctx_list sysctl_tree; 149 struct sysctl_oid *sysctl_tree_top; 150}; 151 152static struct mtx bounce_lock; 153static int total_bpages; 154static int busdma_zonecount; 155static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 156 157static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 158SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 159 "Total bounce pages"); 160 161#define DMAMAP_LINEAR 0x1 162#define DMAMAP_MBUF 0x2 163#define DMAMAP_UIO 0x4 164#define DMAMAP_CACHE_ALIGNED 0x10 165#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) 166#define DMAMAP_COHERENT 0x8 167struct bus_dmamap { 168 struct bp_list bpages; 169 int pagesneeded; 170 int pagesreserved; 171 bus_dma_tag_t dmat; 172 int flags; 173 void *buffer; 174 int len; 175 STAILQ_ENTRY(bus_dmamap) links; 176 bus_dmamap_callback_t *callback; 177 void *callback_arg; 178 179}; 180 181static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 182static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 183 184static struct mtx busdma_mtx; 185 186MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 187 188static void init_bounce_pages(void *dummy); 189static int alloc_bounce_zone(bus_dma_tag_t dmat); 190static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 191static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 192 int commit); 193static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 194 vm_offset_t vaddr, bus_size_t size); 195static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 196 197/* Default tag, as most drivers provide no parent tag. */ 198bus_dma_tag_t arm_root_dma_tag; 199 200/* 201 * ---------------------------------------------------------------------------- 202 * Begin block of code useful to transplant to other implementations. 203 */ 204 205static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 206 207static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 208static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 209 210/* 211 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 212 * It'll need platform-specific changes if this code is copied. 213 */ 214static int 215dmamap_ctor(void *mem, int size, void *arg, int flags) 216{ 217 bus_dmamap_t map; 218 bus_dma_tag_t dmat; 219 220 map = (bus_dmamap_t)mem; 221 dmat = (bus_dma_tag_t)arg; 222 223 dmat->map_count++; 224 225 map->dmat = dmat; 226 map->flags = 0; 227 STAILQ_INIT(&map->bpages); 228 229 return (0); 230} 231 232/* 233 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 234 * It may need platform-specific changes if this code is copied . 235 */ 236static void 237dmamap_dtor(void *mem, int size, void *arg) 238{ 239 bus_dmamap_t map; 240 241 map = (bus_dmamap_t)mem; 242 243 map->dmat->map_count--; 244} 245 246static void 247busdma_init(void *dummy) 248{ 249 250 /* Create a cache of maps for bus_dmamap_create(). */ 251 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 252 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 253 254 /* Create a cache of buffers in standard (cacheable) memory. */ 255 standard_allocator = busdma_bufalloc_create("buffer", 256 arm_dcache_align, /* minimum_alignment */ 257 NULL, /* uma_alloc func */ 258 NULL, /* uma_free func */ 259 0); /* uma_zcreate_flags */ 260 261 /* 262 * Create a cache of buffers in uncacheable memory, to implement the 263 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 264 */ 265 coherent_allocator = busdma_bufalloc_create("coherent", 266 arm_dcache_align, /* minimum_alignment */ 267 busdma_bufalloc_alloc_uncacheable, 268 busdma_bufalloc_free_uncacheable, 269 0); /* uma_zcreate_flags */ 270} 271 272/* 273 * This init historically used SI_SUB_VM, but now the init code requires 274 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 275 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 276 * SI_SUB_KMEM and SI_ORDER_THIRD. 277 */ 278SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 279 280/* 281 * End block of code useful to transplant to other implementations. 282 * ---------------------------------------------------------------------------- 283 */ 284 285/* 286 * Return true if a match is made. 287 * 288 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 289 * 290 * If paddr is within the bounds of the dma tag then call the filter callback 291 * to check for a match, if there is no filter callback then assume a match. 292 */ 293static int 294run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 295{ 296 int retval; 297 298 retval = 0; 299 300 do { 301 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 302 || ((paddr & (dmat->alignment - 1)) != 0)) 303 && (dmat->filter == NULL 304 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 305 retval = 1; 306 307 dmat = dmat->parent; 308 } while (retval == 0 && dmat != NULL); 309 return (retval); 310} 311 312/* 313 * This routine checks the exclusion zone constraints from a tag against the 314 * physical RAM available on the machine. If a tag specifies an exclusion zone 315 * but there's no RAM in that zone, then we avoid allocating resources to bounce 316 * a request, and we can use any memory allocator (as opposed to needing 317 * kmem_alloc_contig() just because it can allocate pages in an address range). 318 * 319 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 320 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 321 * possibly have RAM at an address higher than the highest address we can 322 * express, so we take a fast out. 323 */ 324static __inline int 325_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 326{ 327 int i; 328 329 if (lowaddr >= BUS_SPACE_MAXADDR) 330 return (0); 331 332 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 333 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 334 || (lowaddr < phys_avail[i] && 335 highaddr > phys_avail[i])) 336 return (1); 337 } 338 return (0); 339} 340 341static __inline struct arm32_dma_range * 342_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 343 bus_addr_t curaddr) 344{ 345 struct arm32_dma_range *dr; 346 int i; 347 348 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 349 if (curaddr >= dr->dr_sysbase && 350 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 351 return (dr); 352 } 353 354 return (NULL); 355} 356/* 357 * Convenience function for manipulating driver locks from busdma (during 358 * busdma_swi, for example). Drivers that don't provide their own locks 359 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 360 * non-mutex locking scheme don't have to use this at all. 361 */ 362void 363busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 364{ 365 struct mtx *dmtx; 366 367 dmtx = (struct mtx *)arg; 368 switch (op) { 369 case BUS_DMA_LOCK: 370 mtx_lock(dmtx); 371 break; 372 case BUS_DMA_UNLOCK: 373 mtx_unlock(dmtx); 374 break; 375 default: 376 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 377 } 378} 379 380/* 381 * dflt_lock should never get called. It gets put into the dma tag when 382 * lockfunc == NULL, which is only valid if the maps that are associated 383 * with the tag are meant to never be defered. 384 * XXX Should have a way to identify which driver is responsible here. 385 */ 386static void 387dflt_lock(void *arg, bus_dma_lock_op_t op) 388{ 389#ifdef INVARIANTS 390 panic("driver error: busdma dflt_lock called"); 391#else 392 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 393#endif 394} 395 396/* 397 * Allocate a device specific dma_tag. 398 */ 399#define SEG_NB 1024 400 401int 402bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 403 bus_addr_t boundary, bus_addr_t lowaddr, 404 bus_addr_t highaddr, bus_dma_filter_t *filter, 405 void *filterarg, bus_size_t maxsize, int nsegments, 406 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 407 void *lockfuncarg, bus_dma_tag_t *dmat) 408{ 409 bus_dma_tag_t newtag; 410 int error = 0; 411 /* Return a NULL tag on failure */ 412 *dmat = NULL; 413 if (!parent) 414 parent = arm_root_dma_tag; 415 416 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 417 if (newtag == NULL) { 418 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 419 __func__, newtag, 0, error); 420 return (ENOMEM); 421 } 422 423 newtag->parent = parent; 424 newtag->alignment = alignment ? alignment : 1; 425 newtag->boundary = boundary; 426 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 427 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 428 newtag->filter = filter; 429 newtag->filterarg = filterarg; 430 newtag->maxsize = maxsize; 431 newtag->nsegments = nsegments; 432 newtag->maxsegsz = maxsegsz; 433 newtag->flags = flags; 434 newtag->ref_count = 1; /* Count ourself */ 435 newtag->map_count = 0; 436 newtag->ranges = bus_dma_get_range(); 437 newtag->_nranges = bus_dma_get_range_nb(); 438 if (lockfunc != NULL) { 439 newtag->lockfunc = lockfunc; 440 newtag->lockfuncarg = lockfuncarg; 441 } else { 442 newtag->lockfunc = dflt_lock; 443 newtag->lockfuncarg = NULL; 444 } 445 /* 446 * If all the segments we need fit into the local tagsegs array, set the 447 * pointer now. Otherwise NULL the pointer and an array of segments 448 * will be allocated later, on first use. We don't pre-allocate now 449 * because some tags exist just to pass contraints to children in the 450 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 451 * sure don't want to try to allocate an array for that. 452 */ 453 if (newtag->nsegments <= nitems(newtag->tagsegs)) 454 newtag->segments = newtag->tagsegs; 455 else 456 newtag->segments = NULL; 457 /* 458 * Take into account any restrictions imposed by our parent tag 459 */ 460 if (parent != NULL) { 461 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 462 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 463 if (newtag->boundary == 0) 464 newtag->boundary = parent->boundary; 465 else if (parent->boundary != 0) 466 newtag->boundary = MIN(parent->boundary, 467 newtag->boundary); 468 if ((newtag->filter != NULL) || 469 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 470 newtag->flags |= BUS_DMA_COULD_BOUNCE; 471 if (newtag->filter == NULL) { 472 /* 473 * Short circuit looking at our parent directly 474 * since we have encapsulated all of its information 475 */ 476 newtag->filter = parent->filter; 477 newtag->filterarg = parent->filterarg; 478 newtag->parent = parent->parent; 479 } 480 if (newtag->parent != NULL) 481 atomic_add_int(&parent->ref_count, 1); 482 } 483 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 484 || newtag->alignment > 1) 485 newtag->flags |= BUS_DMA_COULD_BOUNCE; 486 487 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 488 (flags & BUS_DMA_ALLOCNOW) != 0) { 489 struct bounce_zone *bz; 490 491 /* Must bounce */ 492 493 if ((error = alloc_bounce_zone(newtag)) != 0) { 494 free(newtag, M_DEVBUF); 495 return (error); 496 } 497 bz = newtag->bounce_zone; 498 499 if (ptoa(bz->total_bpages) < maxsize) { 500 int pages; 501 502 pages = atop(maxsize) - bz->total_bpages; 503 504 /* Add pages to our bounce pool */ 505 if (alloc_bounce_pages(newtag, pages) < pages) 506 error = ENOMEM; 507 } 508 /* Performed initial allocation */ 509 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 510 } else 511 newtag->bounce_zone = NULL; 512 if (error != 0) 513 free(newtag, M_DEVBUF); 514 else 515 *dmat = newtag; 516 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 517 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 518 519 return (error); 520} 521 522int 523bus_dma_tag_destroy(bus_dma_tag_t dmat) 524{ 525#ifdef KTR 526 bus_dma_tag_t dmat_copy = dmat; 527#endif 528 529 if (dmat != NULL) { 530 531 if (dmat->map_count != 0) 532 return (EBUSY); 533 534 while (dmat != NULL) { 535 bus_dma_tag_t parent; 536 537 parent = dmat->parent; 538 atomic_subtract_int(&dmat->ref_count, 1); 539 if (dmat->ref_count == 0) { 540 if (dmat->segments != NULL && 541 dmat->segments != dmat->tagsegs) 542 free(dmat->segments, M_DEVBUF); 543 free(dmat, M_DEVBUF); 544 /* 545 * Last reference count, so 546 * release our reference 547 * count on our parent. 548 */ 549 dmat = parent; 550 } else 551 dmat = NULL; 552 } 553 } 554 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 555 556 return (0); 557} 558 559#include <sys/kdb.h> 560/* 561 * Allocate a handle for mapping from kva/uva/physical 562 * address space into bus device space. 563 */ 564int 565bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 566{ 567 bus_dmamap_t map; 568 int error = 0; 569 570 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 571 *mapp = map; 572 if (map == NULL) 573 return (ENOMEM); 574 575 /* 576 * If the tag's segments haven't been allocated yet we need to do it 577 * now, because we can't sleep for resources at map load time. 578 */ 579 if (dmat->segments == NULL) { 580 dmat->segments = malloc(dmat->nsegments * 581 sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 582 if (dmat->segments == NULL) { 583 uma_zfree(dmamap_zone, map); 584 *mapp = NULL; 585 return (ENOMEM); 586 } 587 } 588 589 /* 590 * Bouncing might be required if the driver asks for an active 591 * exclusion region, a data alignment that is stricter than 1, and/or 592 * an active address boundary. 593 */ 594 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 595 596 /* Must bounce */ 597 struct bounce_zone *bz; 598 int maxpages; 599 600 if (dmat->bounce_zone == NULL) { 601 if ((error = alloc_bounce_zone(dmat)) != 0) { 602 uma_zfree(dmamap_zone, map); 603 *mapp = NULL; 604 return (error); 605 } 606 } 607 bz = dmat->bounce_zone; 608 609 /* Initialize the new map */ 610 STAILQ_INIT(&((*mapp)->bpages)); 611 612 /* 613 * Attempt to add pages to our pool on a per-instance 614 * basis up to a sane limit. 615 */ 616 maxpages = MAX_BPAGES; 617 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 618 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 619 int pages; 620 621 pages = MAX(atop(dmat->maxsize), 1); 622 pages = MIN(maxpages - bz->total_bpages, pages); 623 pages = MAX(pages, 1); 624 if (alloc_bounce_pages(dmat, pages) < pages) 625 error = ENOMEM; 626 627 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 628 if (error == 0) 629 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 630 } else { 631 error = 0; 632 } 633 } 634 bz->map_count++; 635 } 636 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 637 __func__, dmat, dmat->flags, error); 638 639 return (0); 640} 641 642/* 643 * Destroy a handle for mapping from kva/uva/physical 644 * address space into bus device space. 645 */ 646int 647bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 648{ 649 650 if (STAILQ_FIRST(&map->bpages) != NULL) { 651 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 652 __func__, dmat, EBUSY); 653 return (EBUSY); 654 } 655 uma_zfree(dmamap_zone, map); 656 if (dmat->bounce_zone) 657 dmat->bounce_zone->map_count--; 658 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 659 return (0); 660} 661 662/* 663 * Allocate a piece of memory that can be efficiently mapped into bus device 664 * space based on the constraints listed in the dma tag. Returns a pointer to 665 * the allocated memory, and a pointer to an associated bus_dmamap. 666 */ 667int 668bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 669 bus_dmamap_t *mapp) 670{ 671 void * vaddr; 672 struct busdma_bufzone *bufzone; 673 busdma_bufalloc_t ba; 674 bus_dmamap_t map; 675 int mflags; 676 vm_memattr_t memattr; 677 678 if (flags & BUS_DMA_NOWAIT) 679 mflags = M_NOWAIT; 680 else 681 mflags = M_WAITOK; 682 683 /* 684 * If the tag's segments haven't been allocated yet we need to do it 685 * now, because we can't sleep for resources at map load time. 686 */ 687 if (dmat->segments == NULL) 688 dmat->segments = malloc(dmat->nsegments * 689 sizeof(*dmat->segments), M_DEVBUF, mflags); 690 691 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 692 if (map == NULL) 693 return (ENOMEM); 694 695 if (flags & BUS_DMA_COHERENT) { 696 memattr = VM_MEMATTR_UNCACHEABLE; 697 ba = coherent_allocator; 698 map->flags |= DMAMAP_COHERENT; 699 } else { 700 memattr = VM_MEMATTR_DEFAULT; 701 ba = standard_allocator; 702 } 703 /* All buffers we allocate are cache-aligned. */ 704 map->flags |= DMAMAP_CACHE_ALIGNED; 705 706 if (flags & BUS_DMA_ZERO) 707 mflags |= M_ZERO; 708 709 /* 710 * Try to find a bufzone in the allocator that holds a cache of buffers 711 * of the right size for this request. If the buffer is too big to be 712 * held in the allocator cache, this returns NULL. 713 */ 714 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 715 716 /* 717 * Allocate the buffer from the uma(9) allocator if... 718 * - It's small enough to be in the allocator (bufzone not NULL). 719 * - The alignment constraint isn't larger than the allocation size 720 * (the allocator aligns buffers to their size boundaries). 721 * - There's no need to handle lowaddr/highaddr exclusion zones. 722 * else allocate non-contiguous pages if... 723 * - The page count that could get allocated doesn't exceed nsegments. 724 * - The alignment constraint isn't larger than a page boundary. 725 * - There are no boundary-crossing constraints. 726 * else allocate a block of contiguous pages because one or more of the 727 * constraints is something that only the contig allocator can fulfill. 728 */ 729 if (bufzone != NULL && dmat->alignment <= bufzone->size && 730 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 731 vaddr = uma_zalloc(bufzone->umazone, mflags); 732 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 733 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 734 vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize, 735 mflags, 0, dmat->lowaddr, memattr); 736 } else { 737 vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize, 738 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 739 memattr); 740 } 741 742 if (vaddr == NULL) { 743 uma_zfree(dmamap_zone, map); 744 map = NULL; 745 } 746 747 *vaddrp = vaddr; 748 *mapp = map; 749 750 return (vaddr == NULL ? ENOMEM : 0); 751} 752 753/* 754 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 755 * its associated map. 756 */ 757void 758bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 759{ 760 struct busdma_bufzone *bufzone; 761 busdma_bufalloc_t ba; 762 763 if (map->flags & DMAMAP_COHERENT) 764 ba = coherent_allocator; 765 else 766 ba = standard_allocator; 767 uma_zfree(dmamap_zone, map); 768 769 /* Be careful not to access map from here on. */ 770 771 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 772 773 if (bufzone != NULL && dmat->alignment <= bufzone->size && 774 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 775 uma_zfree(bufzone->umazone, vaddr); 776 else 777 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); 778} 779 780static int 781_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 782 void *buf, bus_size_t buflen, int flags) 783{ 784 vm_offset_t vaddr; 785 vm_offset_t vendaddr; 786 bus_addr_t paddr; 787 788 if ((map->pagesneeded == 0)) { 789 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 790 dmat->lowaddr, dmat->boundary, dmat->alignment); 791 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 792 map, map->pagesneeded); 793 /* 794 * Count the number of bounce pages 795 * needed in order to complete this transfer 796 */ 797 vaddr = trunc_page((vm_offset_t)buf); 798 vendaddr = (vm_offset_t)buf + buflen; 799 800 while (vaddr < vendaddr) { 801 if (__predict_true(pmap == pmap_kernel())) 802 paddr = pmap_kextract(vaddr); 803 else 804 paddr = pmap_extract(pmap, vaddr); 805 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 806 run_filter(dmat, paddr) != 0) 807 map->pagesneeded++; 808 vaddr += PAGE_SIZE; 809 } 810 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 811 } 812 813 /* Reserve Necessary Bounce Pages */ 814 if (map->pagesneeded != 0) { 815 mtx_lock(&bounce_lock); 816 if (flags & BUS_DMA_NOWAIT) { 817 if (reserve_bounce_pages(dmat, map, 0) != 0) { 818 mtx_unlock(&bounce_lock); 819 return (ENOMEM); 820 } 821 } else { 822 if (reserve_bounce_pages(dmat, map, 1) != 0) { 823 /* Queue us for resources */ 824 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 825 map, links); 826 mtx_unlock(&bounce_lock); 827 return (EINPROGRESS); 828 } 829 } 830 mtx_unlock(&bounce_lock); 831 } 832 833 return (0); 834} 835 836/* 837 * Utility function to load a linear buffer. lastaddrp holds state 838 * between invocations (for multiple-buffer loads). segp contains 839 * the starting segment on entrance, and the ending segment on exit. 840 * first indicates if this is the first invocation of this function. 841 */ 842static __inline int 843bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 844 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 845 int flags, vm_offset_t *lastaddrp, int *segp) 846{ 847 bus_size_t sgsize; 848 bus_addr_t curaddr, lastaddr, baddr, bmask; 849 vm_offset_t vaddr = (vm_offset_t)buf; 850 int seg; 851 int error = 0; 852 853 lastaddr = *lastaddrp; 854 bmask = ~(dmat->boundary - 1); 855 856 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 857 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, 858 flags); 859 if (error) 860 return (error); 861 } 862 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 863 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 864 865 for (seg = *segp; buflen > 0 ; ) { 866 /* 867 * Get the physical address for this segment. 868 */ 869 if (__predict_true(pmap == pmap_kernel())) { 870 curaddr = pmap_kextract(vaddr); 871 } else { 872 curaddr = pmap_extract(pmap, vaddr); 873 map->flags &= ~DMAMAP_COHERENT; 874 } 875 876 /* 877 * Compute the segment size, and adjust counts. 878 */ 879 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 880 if (sgsize > dmat->maxsegsz) 881 sgsize = dmat->maxsegsz; 882 if (buflen < sgsize) 883 sgsize = buflen; 884 885 /* 886 * Make sure we don't cross any boundaries. 887 */ 888 if (dmat->boundary > 0) { 889 baddr = (curaddr + dmat->boundary) & bmask; 890 if (sgsize > (baddr - curaddr)) 891 sgsize = (baddr - curaddr); 892 } 893 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 894 map->pagesneeded != 0 && run_filter(dmat, curaddr)) 895 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 896 897 if (dmat->ranges) { 898 struct arm32_dma_range *dr; 899 900 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 901 curaddr); 902 if (dr == NULL) 903 return (EINVAL); 904 /* 905 * In a valid DMA range. Translate the physical 906 * memory address to an address in the DMA window. 907 */ 908 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 909 910 } 911 912 /* 913 * Insert chunk into a segment, coalescing with 914 * the previous segment if possible. 915 */ 916 if (seg >= 0 && curaddr == lastaddr && 917 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 918 (dmat->boundary == 0 || 919 (segs[seg].ds_addr & bmask) == 920 (curaddr & bmask))) { 921 segs[seg].ds_len += sgsize; 922 goto segdone; 923 } else { 924 if (++seg >= dmat->nsegments) 925 break; 926 segs[seg].ds_addr = curaddr; 927 segs[seg].ds_len = sgsize; 928 } 929 if (error) 930 break; 931segdone: 932 lastaddr = curaddr + sgsize; 933 vaddr += sgsize; 934 buflen -= sgsize; 935 } 936 937 *segp = seg; 938 *lastaddrp = lastaddr; 939 940 /* 941 * Did we fit? 942 */ 943 if (buflen != 0) 944 error = EFBIG; /* XXX better return value here? */ 945 return (error); 946} 947 948/* 949 * Map the buffer buf into bus space using the dmamap map. 950 */ 951int 952bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 953 bus_size_t buflen, bus_dmamap_callback_t *callback, 954 void *callback_arg, int flags) 955{ 956 vm_offset_t lastaddr = 0; 957 int error, nsegs = -1; 958 959 KASSERT(dmat != NULL, ("dmatag is NULL")); 960 KASSERT(map != NULL, ("dmamap is NULL")); 961 map->callback = callback; 962 map->callback_arg = callback_arg; 963 map->flags &= ~DMAMAP_TYPE_MASK; 964 map->flags |= DMAMAP_LINEAR; 965 map->buffer = buf; 966 map->len = buflen; 967 error = bus_dmamap_load_buffer(dmat, 968 dmat->segments, map, buf, buflen, kernel_pmap, 969 flags, &lastaddr, &nsegs); 970 if (error == EINPROGRESS) 971 return (error); 972 if (error) 973 (*callback)(callback_arg, NULL, 0, error); 974 else 975 (*callback)(callback_arg, dmat->segments, nsegs + 1, error); 976 977 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 978 __func__, dmat, dmat->flags, nsegs + 1, error); 979 980 return (error); 981} 982 983/* 984 * Like bus_dmamap_load(), but for mbufs. 985 * 986 * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs. 987 * 988 * We know that the way the system allocates and uses mbufs implies that we can 989 * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line 990 * flushes. Even though the flush may reference the data area within the mbuf 991 * that isn't aligned to a cache line, we know the overall mbuf itself is 992 * properly aligned, and we know that the CPU will not touch the header fields 993 * before the data area while the DMA is in progress. 994 */ 995int 996bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 997 bus_dmamap_callback2_t *callback, void *callback_arg, 998 int flags) 999{ 1000 int nsegs = -1, error = 0; 1001 1002 M_ASSERTPKTHDR(m0); 1003 1004 flags |= BUS_DMA_NOWAIT; 1005 map->flags &= ~DMAMAP_TYPE_MASK; 1006 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1007 map->buffer = m0; 1008 map->len = 0; 1009 if (m0->m_pkthdr.len <= dmat->maxsize) { 1010 vm_offset_t lastaddr = 0; 1011 struct mbuf *m; 1012 1013 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1014 if (m->m_len > 0) { 1015 error = bus_dmamap_load_buffer(dmat, 1016 dmat->segments, map, m->m_data, m->m_len, 1017 pmap_kernel(), flags, &lastaddr, &nsegs); 1018 map->len += m->m_len; 1019 } 1020 } 1021 } else { 1022 error = EINVAL; 1023 } 1024 1025 if (error) { 1026 /* 1027 * force "no valid mappings" on error in callback. 1028 */ 1029 (*callback)(callback_arg, NULL, 0, 0, error); 1030 } else { 1031 (*callback)(callback_arg, dmat->segments, nsegs + 1, 1032 m0->m_pkthdr.len, error); 1033 } 1034 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1035 __func__, dmat, dmat->flags, error, nsegs + 1); 1036 1037 return (error); 1038} 1039 1040int 1041bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 1042 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 1043 int flags) 1044{ 1045 int error = 0; 1046 M_ASSERTPKTHDR(m0); 1047 1048 flags |= BUS_DMA_NOWAIT; 1049 *nsegs = -1; 1050 map->flags &= ~DMAMAP_TYPE_MASK; 1051 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1052 map->buffer = m0; 1053 map->len = 0; 1054 if (m0->m_pkthdr.len <= dmat->maxsize) { 1055 vm_offset_t lastaddr = 0; 1056 struct mbuf *m; 1057 1058 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1059 if (m->m_len > 0) { 1060 error = bus_dmamap_load_buffer(dmat, segs, map, 1061 m->m_data, m->m_len, 1062 pmap_kernel(), flags, &lastaddr, 1063 nsegs); 1064 map->len += m->m_len; 1065 } 1066 } 1067 } else { 1068 error = EINVAL; 1069 } 1070 1071 /* XXX FIXME: Having to increment nsegs is really annoying */ 1072 ++*nsegs; 1073 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1074 __func__, dmat, dmat->flags, error, *nsegs); 1075 return (error); 1076} 1077 1078/* 1079 * Like bus_dmamap_load(), but for uios. 1080 */ 1081int 1082bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 1083 bus_dmamap_callback2_t *callback, void *callback_arg, 1084 int flags) 1085{ 1086 vm_offset_t lastaddr = 0; 1087 int nsegs, i, error; 1088 bus_size_t resid; 1089 struct iovec *iov; 1090 struct pmap *pmap; 1091 1092 resid = uio->uio_resid; 1093 iov = uio->uio_iov; 1094 map->flags &= ~DMAMAP_TYPE_MASK; 1095 map->flags |= DMAMAP_UIO; 1096 map->buffer = uio; 1097 map->len = 0; 1098 1099 if (uio->uio_segflg == UIO_USERSPACE) { 1100 KASSERT(uio->uio_td != NULL, 1101 ("bus_dmamap_load_uio: USERSPACE but no proc")); 1102 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 1103 } else 1104 pmap = kernel_pmap; 1105 1106 error = 0; 1107 nsegs = -1; 1108 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 1109 /* 1110 * Now at the first iovec to load. Load each iovec 1111 * until we have exhausted the residual count. 1112 */ 1113 bus_size_t minlen = 1114 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1115 caddr_t addr = (caddr_t) iov[i].iov_base; 1116 1117 if (minlen > 0) { 1118 error = bus_dmamap_load_buffer(dmat, dmat->segments, 1119 map, addr, minlen, pmap, flags, &lastaddr, &nsegs); 1120 1121 map->len += minlen; 1122 resid -= minlen; 1123 } 1124 } 1125 1126 if (error) { 1127 /* 1128 * force "no valid mappings" on error in callback. 1129 */ 1130 (*callback)(callback_arg, NULL, 0, 0, error); 1131 } else { 1132 (*callback)(callback_arg, dmat->segments, nsegs+1, 1133 uio->uio_resid, error); 1134 } 1135 1136 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1137 __func__, dmat, dmat->flags, error, nsegs + 1); 1138 return (error); 1139} 1140 1141/* 1142 * Release the mapping held by map. 1143 */ 1144void 1145_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1146{ 1147 struct bounce_page *bpage; 1148 1149 map->flags &= ~DMAMAP_TYPE_MASK; 1150 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1151 STAILQ_REMOVE_HEAD(&map->bpages, links); 1152 free_bounce_page(dmat, bpage); 1153 } 1154 return; 1155} 1156 1157static void 1158bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned) 1159{ 1160 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1161 register_t s; 1162 int partial; 1163 1164 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1165 cpu_dcache_wb_range((vm_offset_t)buf, len); 1166 cpu_l2cache_wb_range((vm_offset_t)buf, len); 1167 } 1168 1169 /* 1170 * If the caller promises the buffer is properly aligned to a cache line 1171 * (even if the call parms make it look like it isn't) we can avoid 1172 * attempting to preserve the non-DMA part of the cache line in the 1173 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1174 * 1175 * This covers the case of mbufs, where we know how they're aligned and 1176 * know the CPU doesn't touch the header in front of the DMA data area 1177 * during the IO, but it may have touched it right before invoking the 1178 * sync, so a PREREAD writeback is required. 1179 * 1180 * It also handles buffers we created in bus_dmamem_alloc(), which are 1181 * always aligned and padded to cache line size even if the IO length 1182 * isn't a multiple of cache line size. In this case the PREREAD 1183 * writeback probably isn't required, but it's harmless. 1184 */ 1185 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1186 1187 if (op & BUS_DMASYNC_PREREAD) { 1188 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1189 cpu_dcache_inv_range((vm_offset_t)buf, len); 1190 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1191 } else { 1192 cpu_dcache_wbinv_range((vm_offset_t)buf, len); 1193 cpu_l2cache_wbinv_range((vm_offset_t)buf, len); 1194 } 1195 } 1196 if (op & BUS_DMASYNC_POSTREAD) { 1197 if (partial && !bufaligned) { 1198 s = intr_disable(); 1199 if ((vm_offset_t)buf & arm_dcache_align_mask) 1200 memcpy(_tmp_cl, (void *)((vm_offset_t)buf & 1201 ~arm_dcache_align_mask), 1202 (vm_offset_t)buf & arm_dcache_align_mask); 1203 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1204 memcpy(_tmp_clend, 1205 (void *)((vm_offset_t)buf + len), 1206 arm_dcache_align - (((vm_offset_t)(buf) + 1207 len) & arm_dcache_align_mask)); 1208 } 1209 cpu_dcache_inv_range((vm_offset_t)buf, len); 1210 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1211 if (partial && !bufaligned) { 1212 if ((vm_offset_t)buf & arm_dcache_align_mask) 1213 memcpy((void *)((vm_offset_t)buf & 1214 ~arm_dcache_align_mask), _tmp_cl, 1215 (vm_offset_t)buf & arm_dcache_align_mask); 1216 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1217 memcpy((void *)((vm_offset_t)buf + len), 1218 _tmp_clend, arm_dcache_align - 1219 (((vm_offset_t)(buf) + len) & 1220 arm_dcache_align_mask)); 1221 intr_restore(s); 1222 } 1223 } 1224} 1225 1226static void 1227_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1228{ 1229 struct bounce_page *bpage; 1230 1231 STAILQ_FOREACH(bpage, &map->bpages, links) { 1232 if (op & BUS_DMASYNC_PREWRITE) { 1233 bcopy((void *)bpage->datavaddr, 1234 (void *)(bpage->vaddr_nocache != 0 ? 1235 bpage->vaddr_nocache : bpage->vaddr), 1236 bpage->datacount); 1237 if (bpage->vaddr_nocache == 0) { 1238 cpu_dcache_wb_range(bpage->vaddr, 1239 bpage->datacount); 1240 cpu_l2cache_wb_range(bpage->vaddr, 1241 bpage->datacount); 1242 } 1243 dmat->bounce_zone->total_bounced++; 1244 } 1245 if (op & BUS_DMASYNC_POSTREAD) { 1246 if (bpage->vaddr_nocache == 0) { 1247 cpu_dcache_inv_range(bpage->vaddr, 1248 bpage->datacount); 1249 cpu_l2cache_inv_range(bpage->vaddr, 1250 bpage->datacount); 1251 } 1252 bcopy((void *)(bpage->vaddr_nocache != 0 ? 1253 bpage->vaddr_nocache : bpage->vaddr), 1254 (void *)bpage->datavaddr, bpage->datacount); 1255 dmat->bounce_zone->total_bounced++; 1256 } 1257 } 1258} 1259 1260static __inline int 1261_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len) 1262{ 1263 struct bounce_page *bpage; 1264 1265 STAILQ_FOREACH(bpage, &map->bpages, links) { 1266 if ((vm_offset_t)buf >= bpage->datavaddr && 1267 (vm_offset_t)buf + len <= bpage->datavaddr + 1268 bpage->datacount) 1269 return (1); 1270 } 1271 return (0); 1272 1273} 1274 1275void 1276_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1277{ 1278 struct mbuf *m; 1279 struct uio *uio; 1280 int resid; 1281 struct iovec *iov; 1282 int bufaligned; 1283 1284 if (op == BUS_DMASYNC_POSTWRITE) 1285 return; 1286 if (map->flags & DMAMAP_COHERENT) 1287 goto drain; 1288 if (STAILQ_FIRST(&map->bpages)) 1289 _bus_dmamap_sync_bp(dmat, map, op); 1290 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1291 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1292 switch(map->flags & DMAMAP_TYPE_MASK) { 1293 case DMAMAP_LINEAR: 1294 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) 1295 bus_dmamap_sync_buf(map->buffer, map->len, op, 1296 bufaligned); 1297 break; 1298 case DMAMAP_MBUF: 1299 m = map->buffer; 1300 while (m) { 1301 if (m->m_len > 0 && 1302 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) 1303 bus_dmamap_sync_buf(m->m_data, m->m_len, op, 1304 bufaligned); 1305 m = m->m_next; 1306 } 1307 break; 1308 case DMAMAP_UIO: 1309 uio = map->buffer; 1310 iov = uio->uio_iov; 1311 resid = uio->uio_resid; 1312 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1313 bus_size_t minlen = resid < iov[i].iov_len ? resid : 1314 iov[i].iov_len; 1315 if (minlen > 0) { 1316 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, 1317 minlen)) 1318 bus_dmamap_sync_buf(iov[i].iov_base, 1319 minlen, op, bufaligned); 1320 resid -= minlen; 1321 } 1322 } 1323 break; 1324 default: 1325 break; 1326 } 1327 1328drain: 1329 1330 cpu_drain_writebuf(); 1331} 1332 1333static void 1334init_bounce_pages(void *dummy __unused) 1335{ 1336 1337 total_bpages = 0; 1338 STAILQ_INIT(&bounce_zone_list); 1339 STAILQ_INIT(&bounce_map_waitinglist); 1340 STAILQ_INIT(&bounce_map_callbacklist); 1341 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1342} 1343SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1344 1345static struct sysctl_ctx_list * 1346busdma_sysctl_tree(struct bounce_zone *bz) 1347{ 1348 return (&bz->sysctl_tree); 1349} 1350 1351static struct sysctl_oid * 1352busdma_sysctl_tree_top(struct bounce_zone *bz) 1353{ 1354 return (bz->sysctl_tree_top); 1355} 1356 1357static int 1358alloc_bounce_zone(bus_dma_tag_t dmat) 1359{ 1360 struct bounce_zone *bz; 1361 1362 /* Check to see if we already have a suitable zone */ 1363 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1364 if ((dmat->alignment <= bz->alignment) 1365 && (dmat->lowaddr >= bz->lowaddr)) { 1366 dmat->bounce_zone = bz; 1367 return (0); 1368 } 1369 } 1370 1371 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1372 M_NOWAIT | M_ZERO)) == NULL) 1373 return (ENOMEM); 1374 1375 STAILQ_INIT(&bz->bounce_page_list); 1376 bz->free_bpages = 0; 1377 bz->reserved_bpages = 0; 1378 bz->active_bpages = 0; 1379 bz->lowaddr = dmat->lowaddr; 1380 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1381 bz->map_count = 0; 1382 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1383 busdma_zonecount++; 1384 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1385 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1386 dmat->bounce_zone = bz; 1387 1388 sysctl_ctx_init(&bz->sysctl_tree); 1389 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1390 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1391 CTLFLAG_RD, 0, ""); 1392 if (bz->sysctl_tree_top == NULL) { 1393 sysctl_ctx_free(&bz->sysctl_tree); 1394 return (0); /* XXX error code? */ 1395 } 1396 1397 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1398 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1399 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1400 "Total bounce pages"); 1401 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1402 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1403 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1404 "Free bounce pages"); 1405 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1406 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1407 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1408 "Reserved bounce pages"); 1409 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1410 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1411 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1412 "Active bounce pages"); 1413 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1414 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1415 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1416 "Total bounce requests"); 1417 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1418 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1419 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1420 "Total bounce requests that were deferred"); 1421 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1422 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1423 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1424 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1425 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1426 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1427 1428 return (0); 1429} 1430 1431static int 1432alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1433{ 1434 struct bounce_zone *bz; 1435 int count; 1436 1437 bz = dmat->bounce_zone; 1438 count = 0; 1439 while (numpages > 0) { 1440 struct bounce_page *bpage; 1441 1442 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1443 M_NOWAIT | M_ZERO); 1444 1445 if (bpage == NULL) 1446 break; 1447 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1448 M_NOWAIT, 0ul, 1449 bz->lowaddr, 1450 PAGE_SIZE, 1451 0); 1452 if (bpage->vaddr == 0) { 1453 free(bpage, M_DEVBUF); 1454 break; 1455 } 1456 bpage->busaddr = pmap_kextract(bpage->vaddr); 1457 bpage->vaddr_nocache = (vm_offset_t)arm_remap_nocache( 1458 (void *)bpage->vaddr, PAGE_SIZE); 1459 mtx_lock(&bounce_lock); 1460 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1461 total_bpages++; 1462 bz->total_bpages++; 1463 bz->free_bpages++; 1464 mtx_unlock(&bounce_lock); 1465 count++; 1466 numpages--; 1467 } 1468 return (count); 1469} 1470 1471static int 1472reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1473{ 1474 struct bounce_zone *bz; 1475 int pages; 1476 1477 mtx_assert(&bounce_lock, MA_OWNED); 1478 bz = dmat->bounce_zone; 1479 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1480 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1481 return (map->pagesneeded - (map->pagesreserved + pages)); 1482 bz->free_bpages -= pages; 1483 bz->reserved_bpages += pages; 1484 map->pagesreserved += pages; 1485 pages = map->pagesneeded - map->pagesreserved; 1486 1487 return (pages); 1488} 1489 1490static bus_addr_t 1491add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1492 bus_size_t size) 1493{ 1494 struct bounce_zone *bz; 1495 struct bounce_page *bpage; 1496 1497 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1498 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1499 1500 bz = dmat->bounce_zone; 1501 if (map->pagesneeded == 0) 1502 panic("add_bounce_page: map doesn't need any pages"); 1503 map->pagesneeded--; 1504 1505 if (map->pagesreserved == 0) 1506 panic("add_bounce_page: map doesn't need any pages"); 1507 map->pagesreserved--; 1508 1509 mtx_lock(&bounce_lock); 1510 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1511 if (bpage == NULL) 1512 panic("add_bounce_page: free page list is empty"); 1513 1514 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1515 bz->reserved_bpages--; 1516 bz->active_bpages++; 1517 mtx_unlock(&bounce_lock); 1518 1519 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1520 /* Page offset needs to be preserved. */ 1521 bpage->vaddr |= vaddr & PAGE_MASK; 1522 bpage->busaddr |= vaddr & PAGE_MASK; 1523 } 1524 bpage->datavaddr = vaddr; 1525 bpage->datacount = size; 1526 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1527 return (bpage->busaddr); 1528} 1529 1530static void 1531free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1532{ 1533 struct bus_dmamap *map; 1534 struct bounce_zone *bz; 1535 1536 bz = dmat->bounce_zone; 1537 bpage->datavaddr = 0; 1538 bpage->datacount = 0; 1539 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1540 /* 1541 * Reset the bounce page to start at offset 0. Other uses 1542 * of this bounce page may need to store a full page of 1543 * data and/or assume it starts on a page boundary. 1544 */ 1545 bpage->vaddr &= ~PAGE_MASK; 1546 bpage->busaddr &= ~PAGE_MASK; 1547 } 1548 1549 mtx_lock(&bounce_lock); 1550 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1551 bz->free_bpages++; 1552 bz->active_bpages--; 1553 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1554 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1555 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1556 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1557 map, links); 1558 busdma_swi_pending = 1; 1559 bz->total_deferred++; 1560 swi_sched(vm_ih, 0); 1561 } 1562 } 1563 mtx_unlock(&bounce_lock); 1564} 1565 1566void 1567busdma_swi(void) 1568{ 1569 bus_dma_tag_t dmat; 1570 struct bus_dmamap *map; 1571 1572 mtx_lock(&bounce_lock); 1573 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1574 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1575 mtx_unlock(&bounce_lock); 1576 dmat = map->dmat; 1577 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1578 bus_dmamap_load(map->dmat, map, map->buffer, map->len, 1579 map->callback, map->callback_arg, /*flags*/0); 1580 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1581 mtx_lock(&bounce_lock); 1582 } 1583 mtx_unlock(&bounce_lock); 1584} 1585