busdma_machdep-v4.c revision 273377
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 273377 2014-10-21 07:31:21Z hselasky $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/interrupt.h> 62#include <sys/lock.h> 63#include <sys/proc.h> 64#include <sys/memdesc.h> 65#include <sys/mutex.h> 66#include <sys/ktr.h> 67#include <sys/kernel.h> 68#include <sys/sysctl.h> 69#include <sys/uio.h> 70 71#include <vm/uma.h> 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74#include <vm/vm_kern.h> 75#include <vm/vm_page.h> 76#include <vm/vm_map.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86 87struct bounce_zone; 88 89struct bus_dma_tag { 90 bus_dma_tag_t parent; 91 bus_size_t alignment; 92 bus_addr_t boundary; 93 bus_addr_t lowaddr; 94 bus_addr_t highaddr; 95 bus_dma_filter_t *filter; 96 void *filterarg; 97 bus_size_t maxsize; 98 u_int nsegments; 99 bus_size_t maxsegsz; 100 int flags; 101 int ref_count; 102 int map_count; 103 bus_dma_lock_t *lockfunc; 104 void *lockfuncarg; 105 /* 106 * DMA range for this tag. If the page doesn't fall within 107 * one of these ranges, an error is returned. The caller 108 * may then decide what to do with the transfer. If the 109 * range pointer is NULL, it is ignored. 110 */ 111 struct arm32_dma_range *ranges; 112 int _nranges; 113 struct bounce_zone *bounce_zone; 114 /* 115 * Most tags need one or two segments, and can use the local tagsegs 116 * array. For tags with a larger limit, we'll allocate a bigger array 117 * on first use. 118 */ 119 bus_dma_segment_t *segments; 120 bus_dma_segment_t tagsegs[2]; 121}; 122 123struct bounce_page { 124 vm_offset_t vaddr; /* kva of bounce buffer */ 125 bus_addr_t busaddr; /* Physical address */ 126 vm_offset_t datavaddr; /* kva of client data */ 127 bus_addr_t dataaddr; /* client physical address */ 128 bus_size_t datacount; /* client data count */ 129 STAILQ_ENTRY(bounce_page) links; 130}; 131 132struct sync_list { 133 vm_offset_t vaddr; /* kva of bounce buffer */ 134 bus_addr_t busaddr; /* Physical address */ 135 bus_size_t datacount; /* client data count */ 136}; 137 138int busdma_swi_pending; 139 140struct bounce_zone { 141 STAILQ_ENTRY(bounce_zone) links; 142 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 143 int total_bpages; 144 int free_bpages; 145 int reserved_bpages; 146 int active_bpages; 147 int total_bounced; 148 int total_deferred; 149 int map_count; 150 bus_size_t alignment; 151 bus_addr_t lowaddr; 152 char zoneid[8]; 153 char lowaddrid[20]; 154 struct sysctl_ctx_list sysctl_tree; 155 struct sysctl_oid *sysctl_tree_top; 156}; 157 158static struct mtx bounce_lock; 159static int total_bpages; 160static int busdma_zonecount; 161static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 162 163static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 164SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 165 "Total bounce pages"); 166 167#define DMAMAP_COHERENT 0x8 168#define DMAMAP_CACHE_ALIGNED 0x10 169 170struct bus_dmamap { 171 struct bp_list bpages; 172 int pagesneeded; 173 int pagesreserved; 174 bus_dma_tag_t dmat; 175 struct memdesc mem; 176 int flags; 177 STAILQ_ENTRY(bus_dmamap) links; 178 bus_dmamap_callback_t *callback; 179 void *callback_arg; 180 int sync_count; 181 struct sync_list *slist; 182}; 183 184static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 185static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 186 187static struct mtx busdma_mtx; 188 189MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 190 191static void init_bounce_pages(void *dummy); 192static int alloc_bounce_zone(bus_dma_tag_t dmat); 193static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 194static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 195 int commit); 196static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 197 vm_offset_t vaddr, bus_addr_t addr, 198 bus_size_t size); 199static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200 201/* Default tag, as most drivers provide no parent tag. */ 202bus_dma_tag_t arm_root_dma_tag; 203 204/* 205 * ---------------------------------------------------------------------------- 206 * Begin block of code useful to transplant to other implementations. 207 */ 208 209static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 210 211static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 212static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 213 214/* 215 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 216 * It'll need platform-specific changes if this code is copied. 217 */ 218static int 219dmamap_ctor(void *mem, int size, void *arg, int flags) 220{ 221 bus_dmamap_t map; 222 bus_dma_tag_t dmat; 223 224 map = (bus_dmamap_t)mem; 225 dmat = (bus_dma_tag_t)arg; 226 227 dmat->map_count++; 228 229 map->dmat = dmat; 230 map->flags = 0; 231 STAILQ_INIT(&map->bpages); 232 233 return (0); 234} 235 236/* 237 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 238 * It may need platform-specific changes if this code is copied . 239 */ 240static void 241dmamap_dtor(void *mem, int size, void *arg) 242{ 243 bus_dmamap_t map; 244 245 map = (bus_dmamap_t)mem; 246 247 map->dmat->map_count--; 248} 249 250static void 251busdma_init(void *dummy) 252{ 253 254 /* Create a cache of maps for bus_dmamap_create(). */ 255 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 256 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 257 258 /* Create a cache of buffers in standard (cacheable) memory. */ 259 standard_allocator = busdma_bufalloc_create("buffer", 260 arm_dcache_align, /* minimum_alignment */ 261 NULL, /* uma_alloc func */ 262 NULL, /* uma_free func */ 263 0); /* uma_zcreate_flags */ 264 265 /* 266 * Create a cache of buffers in uncacheable memory, to implement the 267 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 268 */ 269 coherent_allocator = busdma_bufalloc_create("coherent", 270 arm_dcache_align, /* minimum_alignment */ 271 busdma_bufalloc_alloc_uncacheable, 272 busdma_bufalloc_free_uncacheable, 273 0); /* uma_zcreate_flags */ 274} 275 276/* 277 * This init historically used SI_SUB_VM, but now the init code requires 278 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 279 * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 280 * SI_SUB_KMEM and SI_ORDER_FOURTH. 281 */ 282SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 283 284/* 285 * End block of code useful to transplant to other implementations. 286 * ---------------------------------------------------------------------------- 287 */ 288 289/* 290 * Return true if a match is made. 291 * 292 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 293 * 294 * If paddr is within the bounds of the dma tag then call the filter callback 295 * to check for a match, if there is no filter callback then assume a match. 296 */ 297static int 298run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 299{ 300 int retval; 301 302 retval = 0; 303 304 do { 305 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 306 || ((paddr & (dmat->alignment - 1)) != 0)) 307 && (dmat->filter == NULL 308 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 309 retval = 1; 310 311 dmat = dmat->parent; 312 } while (retval == 0 && dmat != NULL); 313 return (retval); 314} 315 316/* 317 * This routine checks the exclusion zone constraints from a tag against the 318 * physical RAM available on the machine. If a tag specifies an exclusion zone 319 * but there's no RAM in that zone, then we avoid allocating resources to bounce 320 * a request, and we can use any memory allocator (as opposed to needing 321 * kmem_alloc_contig() just because it can allocate pages in an address range). 322 * 323 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 324 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 325 * possibly have RAM at an address higher than the highest address we can 326 * express, so we take a fast out. 327 */ 328static __inline int 329_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 330{ 331 int i; 332 333 if (lowaddr >= BUS_SPACE_MAXADDR) 334 return (0); 335 336 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 337 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 338 || (lowaddr < phys_avail[i] && 339 highaddr > phys_avail[i])) 340 return (1); 341 } 342 return (0); 343} 344 345static __inline struct arm32_dma_range * 346_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 347 bus_addr_t curaddr) 348{ 349 struct arm32_dma_range *dr; 350 int i; 351 352 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 353 if (curaddr >= dr->dr_sysbase && 354 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 355 return (dr); 356 } 357 358 return (NULL); 359} 360/* 361 * Convenience function for manipulating driver locks from busdma (during 362 * busdma_swi, for example). Drivers that don't provide their own locks 363 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 364 * non-mutex locking scheme don't have to use this at all. 365 */ 366void 367busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 368{ 369 struct mtx *dmtx; 370 371 dmtx = (struct mtx *)arg; 372 switch (op) { 373 case BUS_DMA_LOCK: 374 mtx_lock(dmtx); 375 break; 376 case BUS_DMA_UNLOCK: 377 mtx_unlock(dmtx); 378 break; 379 default: 380 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 381 } 382} 383 384/* 385 * dflt_lock should never get called. It gets put into the dma tag when 386 * lockfunc == NULL, which is only valid if the maps that are associated 387 * with the tag are meant to never be defered. 388 * XXX Should have a way to identify which driver is responsible here. 389 */ 390static void 391dflt_lock(void *arg, bus_dma_lock_op_t op) 392{ 393#ifdef INVARIANTS 394 panic("driver error: busdma dflt_lock called"); 395#else 396 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 397#endif 398} 399 400/* 401 * Allocate a device specific dma_tag. 402 */ 403#define SEG_NB 1024 404 405int 406bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 407 bus_addr_t boundary, bus_addr_t lowaddr, 408 bus_addr_t highaddr, bus_dma_filter_t *filter, 409 void *filterarg, bus_size_t maxsize, int nsegments, 410 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 411 void *lockfuncarg, bus_dma_tag_t *dmat) 412{ 413 bus_dma_tag_t newtag; 414 int error = 0; 415 /* Return a NULL tag on failure */ 416 *dmat = NULL; 417 if (!parent) 418 parent = arm_root_dma_tag; 419 420 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 421 if (newtag == NULL) { 422 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 423 __func__, newtag, 0, error); 424 return (ENOMEM); 425 } 426 427 newtag->parent = parent; 428 newtag->alignment = alignment ? alignment : 1; 429 newtag->boundary = boundary; 430 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 431 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 432 newtag->filter = filter; 433 newtag->filterarg = filterarg; 434 newtag->maxsize = maxsize; 435 newtag->nsegments = nsegments; 436 newtag->maxsegsz = maxsegsz; 437 newtag->flags = flags; 438 newtag->ref_count = 1; /* Count ourself */ 439 newtag->map_count = 0; 440 newtag->ranges = bus_dma_get_range(); 441 newtag->_nranges = bus_dma_get_range_nb(); 442 if (lockfunc != NULL) { 443 newtag->lockfunc = lockfunc; 444 newtag->lockfuncarg = lockfuncarg; 445 } else { 446 newtag->lockfunc = dflt_lock; 447 newtag->lockfuncarg = NULL; 448 } 449 /* 450 * If all the segments we need fit into the local tagsegs array, set the 451 * pointer now. Otherwise NULL the pointer and an array of segments 452 * will be allocated later, on first use. We don't pre-allocate now 453 * because some tags exist just to pass contraints to children in the 454 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 455 * sure don't want to try to allocate an array for that. 456 */ 457 if (newtag->nsegments <= nitems(newtag->tagsegs)) 458 newtag->segments = newtag->tagsegs; 459 else 460 newtag->segments = NULL; 461 /* 462 * Take into account any restrictions imposed by our parent tag 463 */ 464 if (parent != NULL) { 465 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 466 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 467 if (newtag->boundary == 0) 468 newtag->boundary = parent->boundary; 469 else if (parent->boundary != 0) 470 newtag->boundary = MIN(parent->boundary, 471 newtag->boundary); 472 if ((newtag->filter != NULL) || 473 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 474 newtag->flags |= BUS_DMA_COULD_BOUNCE; 475 if (newtag->filter == NULL) { 476 /* 477 * Short circuit looking at our parent directly 478 * since we have encapsulated all of its information 479 */ 480 newtag->filter = parent->filter; 481 newtag->filterarg = parent->filterarg; 482 newtag->parent = parent->parent; 483 } 484 if (newtag->parent != NULL) 485 atomic_add_int(&parent->ref_count, 1); 486 } 487 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 488 || newtag->alignment > 1) 489 newtag->flags |= BUS_DMA_COULD_BOUNCE; 490 491 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 492 (flags & BUS_DMA_ALLOCNOW) != 0) { 493 struct bounce_zone *bz; 494 495 /* Must bounce */ 496 497 if ((error = alloc_bounce_zone(newtag)) != 0) { 498 free(newtag, M_DEVBUF); 499 return (error); 500 } 501 bz = newtag->bounce_zone; 502 503 if (ptoa(bz->total_bpages) < maxsize) { 504 int pages; 505 506 pages = atop(maxsize) - bz->total_bpages; 507 508 /* Add pages to our bounce pool */ 509 if (alloc_bounce_pages(newtag, pages) < pages) 510 error = ENOMEM; 511 } 512 /* Performed initial allocation */ 513 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 514 } else 515 newtag->bounce_zone = NULL; 516 if (error != 0) 517 free(newtag, M_DEVBUF); 518 else 519 *dmat = newtag; 520 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 521 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 522 523 return (error); 524} 525 526int 527bus_dma_tag_destroy(bus_dma_tag_t dmat) 528{ 529#ifdef KTR 530 bus_dma_tag_t dmat_copy = dmat; 531#endif 532 533 if (dmat != NULL) { 534 535 if (dmat->map_count != 0) 536 return (EBUSY); 537 538 while (dmat != NULL) { 539 bus_dma_tag_t parent; 540 541 parent = dmat->parent; 542 atomic_subtract_int(&dmat->ref_count, 1); 543 if (dmat->ref_count == 0) { 544 if (dmat->segments != NULL && 545 dmat->segments != dmat->tagsegs) 546 free(dmat->segments, M_DEVBUF); 547 free(dmat, M_DEVBUF); 548 /* 549 * Last reference count, so 550 * release our reference 551 * count on our parent. 552 */ 553 dmat = parent; 554 } else 555 dmat = NULL; 556 } 557 } 558 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 559 560 return (0); 561} 562 563#include <sys/kdb.h> 564/* 565 * Allocate a handle for mapping from kva/uva/physical 566 * address space into bus device space. 567 */ 568int 569bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 570{ 571 struct sync_list *slist; 572 bus_dmamap_t map; 573 int error = 0; 574 575 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 576 if (slist == NULL) 577 return (ENOMEM); 578 579 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 580 *mapp = map; 581 if (map == NULL) { 582 free(slist, M_DEVBUF); 583 return (ENOMEM); 584 } 585 586 /* 587 * If the tag's segments haven't been allocated yet we need to do it 588 * now, because we can't sleep for resources at map load time. 589 */ 590 if (dmat->segments == NULL) { 591 dmat->segments = malloc(dmat->nsegments * 592 sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 593 if (dmat->segments == NULL) { 594 free(slist, M_DEVBUF); 595 uma_zfree(dmamap_zone, map); 596 *mapp = NULL; 597 return (ENOMEM); 598 } 599 } 600 601 /* 602 * Bouncing might be required if the driver asks for an active 603 * exclusion region, a data alignment that is stricter than 1, and/or 604 * an active address boundary. 605 */ 606 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 607 608 /* Must bounce */ 609 struct bounce_zone *bz; 610 int maxpages; 611 612 if (dmat->bounce_zone == NULL) { 613 if ((error = alloc_bounce_zone(dmat)) != 0) { 614 free(slist, M_DEVBUF); 615 uma_zfree(dmamap_zone, map); 616 *mapp = NULL; 617 return (error); 618 } 619 } 620 bz = dmat->bounce_zone; 621 622 /* Initialize the new map */ 623 STAILQ_INIT(&((*mapp)->bpages)); 624 625 /* 626 * Attempt to add pages to our pool on a per-instance 627 * basis up to a sane limit. 628 */ 629 maxpages = MAX_BPAGES; 630 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 631 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 632 int pages; 633 634 pages = MAX(atop(dmat->maxsize), 1); 635 pages = MIN(maxpages - bz->total_bpages, pages); 636 pages = MAX(pages, 1); 637 if (alloc_bounce_pages(dmat, pages) < pages) 638 error = ENOMEM; 639 640 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 641 if (error == 0) 642 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 643 } else { 644 error = 0; 645 } 646 } 647 bz->map_count++; 648 } 649 map->sync_count = 0; 650 map->slist = slist; 651 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 652 __func__, dmat, dmat->flags, error); 653 654 return (0); 655} 656 657/* 658 * Destroy a handle for mapping from kva/uva/physical 659 * address space into bus device space. 660 */ 661int 662bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 663{ 664 665 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 666 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 667 __func__, dmat, EBUSY); 668 return (EBUSY); 669 } 670 free(map->slist, M_DEVBUF); 671 uma_zfree(dmamap_zone, map); 672 if (dmat->bounce_zone) 673 dmat->bounce_zone->map_count--; 674 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 675 return (0); 676} 677 678/* 679 * Allocate a piece of memory that can be efficiently mapped into bus device 680 * space based on the constraints listed in the dma tag. Returns a pointer to 681 * the allocated memory, and a pointer to an associated bus_dmamap. 682 */ 683int 684bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 685 bus_dmamap_t *mapp) 686{ 687 struct sync_list *slist; 688 void * vaddr; 689 struct busdma_bufzone *bufzone; 690 busdma_bufalloc_t ba; 691 bus_dmamap_t map; 692 int mflags; 693 vm_memattr_t memattr; 694 695 if (flags & BUS_DMA_NOWAIT) 696 mflags = M_NOWAIT; 697 else 698 mflags = M_WAITOK; 699 /* 700 * If the tag's segments haven't been allocated yet we need to do it 701 * now, because we can't sleep for resources at map load time. 702 */ 703 if (dmat->segments == NULL) 704 dmat->segments = malloc(dmat->nsegments * 705 sizeof(*dmat->segments), M_DEVBUF, mflags); 706 707 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 708 if (slist == NULL) 709 return (ENOMEM); 710 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 711 if (map == NULL) { 712 free(slist, M_DEVBUF); 713 return (ENOMEM); 714 } 715 if (flags & BUS_DMA_COHERENT) { 716 memattr = VM_MEMATTR_UNCACHEABLE; 717 ba = coherent_allocator; 718 map->flags |= DMAMAP_COHERENT; 719 } else { 720 memattr = VM_MEMATTR_DEFAULT; 721 ba = standard_allocator; 722 } 723 /* All buffers we allocate are cache-aligned. */ 724 map->flags |= DMAMAP_CACHE_ALIGNED; 725 726 if (flags & BUS_DMA_ZERO) 727 mflags |= M_ZERO; 728 729 /* 730 * Try to find a bufzone in the allocator that holds a cache of buffers 731 * of the right size for this request. If the buffer is too big to be 732 * held in the allocator cache, this returns NULL. 733 */ 734 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 735 736 /* 737 * Allocate the buffer from the uma(9) allocator if... 738 * - It's small enough to be in the allocator (bufzone not NULL). 739 * - The alignment constraint isn't larger than the allocation size 740 * (the allocator aligns buffers to their size boundaries). 741 * - There's no need to handle lowaddr/highaddr exclusion zones. 742 * else allocate non-contiguous pages if... 743 * - The page count that could get allocated doesn't exceed nsegments. 744 * - The alignment constraint isn't larger than a page boundary. 745 * - There are no boundary-crossing constraints. 746 * else allocate a block of contiguous pages because one or more of the 747 * constraints is something that only the contig allocator can fulfill. 748 */ 749 if (bufzone != NULL && dmat->alignment <= bufzone->size && 750 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 751 vaddr = uma_zalloc(bufzone->umazone, mflags); 752 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 753 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 754 vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 755 mflags, 0, dmat->lowaddr, memattr); 756 } else { 757 vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 758 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 759 memattr); 760 } 761 if (vaddr == NULL) { 762 free(slist, M_DEVBUF); 763 uma_zfree(dmamap_zone, map); 764 map = NULL; 765 } else { 766 map->slist = slist; 767 map->sync_count = 0; 768 } 769 *vaddrp = vaddr; 770 *mapp = map; 771 772 return (vaddr == NULL ? ENOMEM : 0); 773} 774 775/* 776 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 777 * its associated map. 778 */ 779void 780bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 781{ 782 struct busdma_bufzone *bufzone; 783 busdma_bufalloc_t ba; 784 785 if (map->flags & DMAMAP_COHERENT) 786 ba = coherent_allocator; 787 else 788 ba = standard_allocator; 789 uma_zfree(dmamap_zone, map); 790 791 free(map->slist, M_DEVBUF); 792 /* Be careful not to access map from here on. */ 793 794 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 795 796 if (bufzone != NULL && dmat->alignment <= bufzone->size && 797 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 798 uma_zfree(bufzone->umazone, vaddr); 799 else 800 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 801} 802 803static void 804_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 805 bus_size_t buflen, int flags) 806{ 807 bus_addr_t curaddr; 808 bus_size_t sgsize; 809 810 if (map->pagesneeded == 0) { 811 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 812 dmat->lowaddr, dmat->boundary, dmat->alignment); 813 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 814 map, map->pagesneeded); 815 /* 816 * Count the number of bounce pages 817 * needed in order to complete this transfer 818 */ 819 curaddr = buf; 820 while (buflen != 0) { 821 sgsize = MIN(buflen, dmat->maxsegsz); 822 if (run_filter(dmat, curaddr) != 0) { 823 sgsize = MIN(sgsize, PAGE_SIZE); 824 map->pagesneeded++; 825 } 826 curaddr += sgsize; 827 buflen -= sgsize; 828 } 829 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 830 } 831} 832 833static void 834_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 835 void *buf, bus_size_t buflen, int flags) 836{ 837 vm_offset_t vaddr; 838 vm_offset_t vendaddr; 839 bus_addr_t paddr; 840 841 if (map->pagesneeded == 0) { 842 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 843 dmat->lowaddr, dmat->boundary, dmat->alignment); 844 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 845 map, map->pagesneeded); 846 /* 847 * Count the number of bounce pages 848 * needed in order to complete this transfer 849 */ 850 vaddr = trunc_page((vm_offset_t)buf); 851 vendaddr = (vm_offset_t)buf + buflen; 852 853 while (vaddr < vendaddr) { 854 if (__predict_true(pmap == kernel_pmap)) 855 paddr = pmap_kextract(vaddr); 856 else 857 paddr = pmap_extract(pmap, vaddr); 858 if (run_filter(dmat, paddr) != 0) 859 map->pagesneeded++; 860 vaddr += PAGE_SIZE; 861 } 862 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 863 } 864} 865 866static int 867_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 868{ 869 870 /* Reserve Necessary Bounce Pages */ 871 mtx_lock(&bounce_lock); 872 if (flags & BUS_DMA_NOWAIT) { 873 if (reserve_bounce_pages(dmat, map, 0) != 0) { 874 mtx_unlock(&bounce_lock); 875 return (ENOMEM); 876 } 877 } else { 878 if (reserve_bounce_pages(dmat, map, 1) != 0) { 879 /* Queue us for resources */ 880 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 881 mtx_unlock(&bounce_lock); 882 return (EINPROGRESS); 883 } 884 } 885 mtx_unlock(&bounce_lock); 886 887 return (0); 888} 889 890/* 891 * Add a single contiguous physical range to the segment list. 892 */ 893static int 894_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 895 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 896{ 897 bus_addr_t baddr, bmask; 898 int seg; 899 900 /* 901 * Make sure we don't cross any boundaries. 902 */ 903 bmask = ~(dmat->boundary - 1); 904 if (dmat->boundary > 0) { 905 baddr = (curaddr + dmat->boundary) & bmask; 906 if (sgsize > (baddr - curaddr)) 907 sgsize = (baddr - curaddr); 908 } 909 if (dmat->ranges) { 910 struct arm32_dma_range *dr; 911 912 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 913 curaddr); 914 if (dr == NULL) 915 return (0); 916 /* 917 * In a valid DMA range. Translate the physical 918 * memory address to an address in the DMA window. 919 */ 920 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 921 922 } 923 924 seg = *segp; 925 /* 926 * Insert chunk into a segment, coalescing with 927 * the previous segment if possible. 928 */ 929 if (seg >= 0 && 930 curaddr == segs[seg].ds_addr + segs[seg].ds_len && 931 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 932 (dmat->boundary == 0 || 933 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 934 segs[seg].ds_len += sgsize; 935 } else { 936 if (++seg >= dmat->nsegments) 937 return (0); 938 segs[seg].ds_addr = curaddr; 939 segs[seg].ds_len = sgsize; 940 } 941 *segp = seg; 942 return (sgsize); 943} 944 945/* 946 * Utility function to load a physical buffer. segp contains 947 * the starting segment on entrace, and the ending segment on exit. 948 */ 949int 950_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 951 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 952{ 953 bus_size_t sgsize; 954 bus_addr_t curaddr; 955 int error; 956 957 if (segs == NULL) 958 segs = dmat->segments; 959 960 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 961 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 962 if (map->pagesneeded != 0) { 963 error = _bus_dmamap_reserve_pages(dmat, map, flags); 964 if (error) 965 return (error); 966 } 967 } 968 969 while (buflen > 0) { 970 curaddr = buf; 971 sgsize = MIN(buflen, dmat->maxsegsz); 972 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 973 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 974 sgsize = MIN(sgsize, PAGE_SIZE); 975 curaddr = add_bounce_page(dmat, map, 0, curaddr, 976 sgsize); 977 } 978 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 979 segp); 980 if (sgsize == 0) 981 break; 982 buf += sgsize; 983 buflen -= sgsize; 984 } 985 986 /* 987 * Did we fit? 988 */ 989 if (buflen != 0) { 990 _bus_dmamap_unload(dmat, map); 991 return (EFBIG); /* XXX better return value here? */ 992 } 993 return (0); 994} 995 996int 997_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 998 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 999 bus_dma_segment_t *segs, int *segp) 1000{ 1001 1002 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1003 segs, segp)); 1004} 1005 1006/* 1007 * Utility function to load a linear buffer. segp contains 1008 * the starting segment on entrance, and the ending segment on exit. 1009 */ 1010int 1011_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1012 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1013 int *segp) 1014{ 1015 bus_size_t sgsize; 1016 bus_addr_t curaddr; 1017 struct sync_list *sl; 1018 vm_offset_t vaddr = (vm_offset_t)buf; 1019 int error = 0; 1020 1021 if (segs == NULL) 1022 segs = dmat->segments; 1023 if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1024 map->flags |= DMAMAP_CACHE_ALIGNED; 1025 1026 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1027 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1028 if (map->pagesneeded != 0) { 1029 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1030 if (error) 1031 return (error); 1032 } 1033 } 1034 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1035 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1036 1037 while (buflen > 0) { 1038 /* 1039 * Get the physical address for this segment. 1040 */ 1041 if (__predict_true(pmap == kernel_pmap)) { 1042 curaddr = pmap_kextract(vaddr); 1043 } else { 1044 curaddr = pmap_extract(pmap, vaddr); 1045 map->flags &= ~DMAMAP_COHERENT; 1046 } 1047 1048 /* 1049 * Compute the segment size, and adjust counts. 1050 */ 1051 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 1052 if (sgsize > dmat->maxsegsz) 1053 sgsize = dmat->maxsegsz; 1054 if (buflen < sgsize) 1055 sgsize = buflen; 1056 1057 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1058 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1059 curaddr = add_bounce_page(dmat, map, vaddr, curaddr, 1060 sgsize); 1061 } else { 1062 sl = &map->slist[map->sync_count - 1]; 1063 if (map->sync_count == 0 || 1064 vaddr != sl->vaddr + sl->datacount) { 1065 if (++map->sync_count > dmat->nsegments) 1066 goto cleanup; 1067 sl++; 1068 sl->vaddr = vaddr; 1069 sl->datacount = sgsize; 1070 sl->busaddr = curaddr; 1071 } else 1072 sl->datacount += sgsize; 1073 } 1074 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1075 segp); 1076 if (sgsize == 0) 1077 break; 1078 vaddr += sgsize; 1079 buflen -= sgsize; 1080 } 1081 1082cleanup: 1083 /* 1084 * Did we fit? 1085 */ 1086 if (buflen != 0) { 1087 _bus_dmamap_unload(dmat, map); 1088 return (EFBIG); /* XXX better return value here? */ 1089 } 1090 return (0); 1091} 1092 1093void 1094__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 1095 struct memdesc *mem, bus_dmamap_callback_t *callback, 1096 void *callback_arg) 1097{ 1098 1099 KASSERT(dmat != NULL, ("dmatag is NULL")); 1100 KASSERT(map != NULL, ("dmamap is NULL")); 1101 map->mem = *mem; 1102 map->callback = callback; 1103 map->callback_arg = callback_arg; 1104} 1105 1106bus_dma_segment_t * 1107_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1108 bus_dma_segment_t *segs, int nsegs, int error) 1109{ 1110 1111 if (segs == NULL) 1112 segs = dmat->segments; 1113 return (segs); 1114} 1115 1116/* 1117 * Release the mapping held by map. 1118 */ 1119void 1120_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1121{ 1122 struct bounce_page *bpage; 1123 1124 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1125 STAILQ_REMOVE_HEAD(&map->bpages, links); 1126 free_bounce_page(dmat, bpage); 1127 } 1128 map->sync_count = 0; 1129 return; 1130} 1131 1132static void 1133bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1134 int bufaligned) 1135{ 1136 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1137 register_t s; 1138 int partial; 1139 1140 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1141 cpu_dcache_wb_range(buf, len); 1142 cpu_l2cache_wb_range(buf, len); 1143 } 1144 1145 /* 1146 * If the caller promises the buffer is properly aligned to a cache line 1147 * (even if the call parms make it look like it isn't) we can avoid 1148 * attempting to preserve the non-DMA part of the cache line in the 1149 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1150 * 1151 * This covers the case of mbufs, where we know how they're aligned and 1152 * know the CPU doesn't touch the header in front of the DMA data area 1153 * during the IO, but it may have touched it right before invoking the 1154 * sync, so a PREREAD writeback is required. 1155 * 1156 * It also handles buffers we created in bus_dmamem_alloc(), which are 1157 * always aligned and padded to cache line size even if the IO length 1158 * isn't a multiple of cache line size. In this case the PREREAD 1159 * writeback probably isn't required, but it's harmless. 1160 */ 1161 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1162 1163 if (op & BUS_DMASYNC_PREREAD) { 1164 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1165 cpu_dcache_inv_range(buf, len); 1166 cpu_l2cache_inv_range(buf, len); 1167 } else { 1168 cpu_dcache_wbinv_range(buf, len); 1169 cpu_l2cache_wbinv_range(buf, len); 1170 } 1171 } 1172 if (op & BUS_DMASYNC_POSTREAD) { 1173 if (partial && !bufaligned) { 1174 s = intr_disable(); 1175 if (buf & arm_dcache_align_mask) 1176 memcpy(_tmp_cl, (void *)(buf & 1177 ~arm_dcache_align_mask), 1178 buf & arm_dcache_align_mask); 1179 if ((buf + len) & arm_dcache_align_mask) 1180 memcpy(_tmp_clend, 1181 (void *)(buf + len), 1182 arm_dcache_align - 1183 ((buf + len) & arm_dcache_align_mask)); 1184 } 1185 cpu_dcache_inv_range(buf, len); 1186 cpu_l2cache_inv_range(buf, len); 1187 if (partial && !bufaligned) { 1188 if (buf & arm_dcache_align_mask) 1189 memcpy((void *)(buf & 1190 ~arm_dcache_align_mask), _tmp_cl, 1191 buf & arm_dcache_align_mask); 1192 if ((buf + len) & arm_dcache_align_mask) 1193 memcpy((void *)(buf + len), 1194 _tmp_clend, arm_dcache_align - 1195 ((buf + len) & arm_dcache_align_mask)); 1196 intr_restore(s); 1197 } 1198 } 1199} 1200 1201static void 1202_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1203{ 1204 struct bounce_page *bpage; 1205 1206 STAILQ_FOREACH(bpage, &map->bpages, links) { 1207 if (op & BUS_DMASYNC_PREWRITE) { 1208 if (bpage->datavaddr != 0) 1209 bcopy((void *)bpage->datavaddr, 1210 (void *)bpage->vaddr, bpage->datacount); 1211 else 1212 physcopyout(bpage->dataaddr, 1213 (void *)bpage->vaddr,bpage->datacount); 1214 cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1215 cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1216 dmat->bounce_zone->total_bounced++; 1217 } 1218 if (op & BUS_DMASYNC_POSTREAD) { 1219 cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1220 cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1221 if (bpage->datavaddr != 0) 1222 bcopy((void *)bpage->vaddr, 1223 (void *)bpage->datavaddr, bpage->datacount); 1224 else 1225 physcopyin((void *)bpage->vaddr, 1226 bpage->dataaddr, bpage->datacount); 1227 dmat->bounce_zone->total_bounced++; 1228 } 1229 } 1230} 1231 1232void 1233_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1234{ 1235 struct sync_list *sl, *end; 1236 int bufaligned; 1237 1238 if (op == BUS_DMASYNC_POSTWRITE) 1239 return; 1240 if (map->flags & DMAMAP_COHERENT) 1241 goto drain; 1242 if (STAILQ_FIRST(&map->bpages)) 1243 _bus_dmamap_sync_bp(dmat, map, op); 1244 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1245 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1246 if (map->sync_count) { 1247 end = &map->slist[map->sync_count]; 1248 for (sl = &map->slist[0]; sl != end; sl++) 1249 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, 1250 bufaligned); 1251 } 1252 1253drain: 1254 1255 cpu_drain_writebuf(); 1256} 1257 1258static void 1259init_bounce_pages(void *dummy __unused) 1260{ 1261 1262 total_bpages = 0; 1263 STAILQ_INIT(&bounce_zone_list); 1264 STAILQ_INIT(&bounce_map_waitinglist); 1265 STAILQ_INIT(&bounce_map_callbacklist); 1266 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1267} 1268SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1269 1270static struct sysctl_ctx_list * 1271busdma_sysctl_tree(struct bounce_zone *bz) 1272{ 1273 return (&bz->sysctl_tree); 1274} 1275 1276static struct sysctl_oid * 1277busdma_sysctl_tree_top(struct bounce_zone *bz) 1278{ 1279 return (bz->sysctl_tree_top); 1280} 1281 1282static int 1283alloc_bounce_zone(bus_dma_tag_t dmat) 1284{ 1285 struct bounce_zone *bz; 1286 1287 /* Check to see if we already have a suitable zone */ 1288 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1289 if ((dmat->alignment <= bz->alignment) 1290 && (dmat->lowaddr >= bz->lowaddr)) { 1291 dmat->bounce_zone = bz; 1292 return (0); 1293 } 1294 } 1295 1296 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1297 M_NOWAIT | M_ZERO)) == NULL) 1298 return (ENOMEM); 1299 1300 STAILQ_INIT(&bz->bounce_page_list); 1301 bz->free_bpages = 0; 1302 bz->reserved_bpages = 0; 1303 bz->active_bpages = 0; 1304 bz->lowaddr = dmat->lowaddr; 1305 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1306 bz->map_count = 0; 1307 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1308 busdma_zonecount++; 1309 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1310 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1311 dmat->bounce_zone = bz; 1312 1313 sysctl_ctx_init(&bz->sysctl_tree); 1314 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1315 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1316 CTLFLAG_RD, 0, ""); 1317 if (bz->sysctl_tree_top == NULL) { 1318 sysctl_ctx_free(&bz->sysctl_tree); 1319 return (0); /* XXX error code? */ 1320 } 1321 1322 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1323 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1324 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1325 "Total bounce pages"); 1326 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1327 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1328 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1329 "Free bounce pages"); 1330 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1331 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1332 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1333 "Reserved bounce pages"); 1334 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1335 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1336 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1337 "Active bounce pages"); 1338 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1339 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1340 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1341 "Total bounce requests"); 1342 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1343 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1344 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1345 "Total bounce requests that were deferred"); 1346 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1347 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1348 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1349 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1350 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1351 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1352 1353 return (0); 1354} 1355 1356static int 1357alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1358{ 1359 struct bounce_zone *bz; 1360 int count; 1361 1362 bz = dmat->bounce_zone; 1363 count = 0; 1364 while (numpages > 0) { 1365 struct bounce_page *bpage; 1366 1367 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1368 M_NOWAIT | M_ZERO); 1369 1370 if (bpage == NULL) 1371 break; 1372 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1373 M_NOWAIT, 0ul, 1374 bz->lowaddr, 1375 PAGE_SIZE, 1376 0); 1377 if (bpage->vaddr == 0) { 1378 free(bpage, M_DEVBUF); 1379 break; 1380 } 1381 bpage->busaddr = pmap_kextract(bpage->vaddr); 1382 mtx_lock(&bounce_lock); 1383 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1384 total_bpages++; 1385 bz->total_bpages++; 1386 bz->free_bpages++; 1387 mtx_unlock(&bounce_lock); 1388 count++; 1389 numpages--; 1390 } 1391 return (count); 1392} 1393 1394static int 1395reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1396{ 1397 struct bounce_zone *bz; 1398 int pages; 1399 1400 mtx_assert(&bounce_lock, MA_OWNED); 1401 bz = dmat->bounce_zone; 1402 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1403 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1404 return (map->pagesneeded - (map->pagesreserved + pages)); 1405 bz->free_bpages -= pages; 1406 bz->reserved_bpages += pages; 1407 map->pagesreserved += pages; 1408 pages = map->pagesneeded - map->pagesreserved; 1409 1410 return (pages); 1411} 1412 1413static bus_addr_t 1414add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1415 bus_addr_t addr, bus_size_t size) 1416{ 1417 struct bounce_zone *bz; 1418 struct bounce_page *bpage; 1419 1420 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1421 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1422 1423 bz = dmat->bounce_zone; 1424 if (map->pagesneeded == 0) 1425 panic("add_bounce_page: map doesn't need any pages"); 1426 map->pagesneeded--; 1427 1428 if (map->pagesreserved == 0) 1429 panic("add_bounce_page: map doesn't need any pages"); 1430 map->pagesreserved--; 1431 1432 mtx_lock(&bounce_lock); 1433 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1434 if (bpage == NULL) 1435 panic("add_bounce_page: free page list is empty"); 1436 1437 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1438 bz->reserved_bpages--; 1439 bz->active_bpages++; 1440 mtx_unlock(&bounce_lock); 1441 1442 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1443 /* Page offset needs to be preserved. */ 1444 bpage->vaddr |= vaddr & PAGE_MASK; 1445 bpage->busaddr |= vaddr & PAGE_MASK; 1446 } 1447 bpage->datavaddr = vaddr; 1448 bpage->dataaddr = addr; 1449 bpage->datacount = size; 1450 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1451 return (bpage->busaddr); 1452} 1453 1454static void 1455free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1456{ 1457 struct bus_dmamap *map; 1458 struct bounce_zone *bz; 1459 1460 bz = dmat->bounce_zone; 1461 bpage->datavaddr = 0; 1462 bpage->datacount = 0; 1463 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1464 /* 1465 * Reset the bounce page to start at offset 0. Other uses 1466 * of this bounce page may need to store a full page of 1467 * data and/or assume it starts on a page boundary. 1468 */ 1469 bpage->vaddr &= ~PAGE_MASK; 1470 bpage->busaddr &= ~PAGE_MASK; 1471 } 1472 1473 mtx_lock(&bounce_lock); 1474 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1475 bz->free_bpages++; 1476 bz->active_bpages--; 1477 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1478 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1479 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1480 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1481 map, links); 1482 busdma_swi_pending = 1; 1483 bz->total_deferred++; 1484 swi_sched(vm_ih, 0); 1485 } 1486 } 1487 mtx_unlock(&bounce_lock); 1488} 1489 1490void 1491busdma_swi(void) 1492{ 1493 bus_dma_tag_t dmat; 1494 struct bus_dmamap *map; 1495 1496 mtx_lock(&bounce_lock); 1497 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1498 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1499 mtx_unlock(&bounce_lock); 1500 dmat = map->dmat; 1501 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1502 bus_dmamap_load_mem(map->dmat, map, &map->mem, 1503 map->callback, map->callback_arg, BUS_DMA_WAITOK); 1504 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1505 mtx_lock(&bounce_lock); 1506 } 1507 mtx_unlock(&bounce_lock); 1508} 1509