busdma_machdep-v4.c revision 289675
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289675 2015-10-21 04:53:34Z jah $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/interrupt.h> 62#include <sys/lock.h> 63#include <sys/proc.h> 64#include <sys/memdesc.h> 65#include <sys/mutex.h> 66#include <sys/ktr.h> 67#include <sys/kernel.h> 68#include <sys/sysctl.h> 69#include <sys/uio.h> 70 71#include <vm/uma.h> 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74#include <vm/vm_kern.h> 75#include <vm/vm_page.h> 76#include <vm/vm_map.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86 87struct bounce_zone; 88 89struct bus_dma_tag { 90 bus_dma_tag_t parent; 91 bus_size_t alignment; 92 bus_addr_t boundary; 93 bus_addr_t lowaddr; 94 bus_addr_t highaddr; 95 bus_dma_filter_t *filter; 96 void *filterarg; 97 bus_size_t maxsize; 98 u_int nsegments; 99 bus_size_t maxsegsz; 100 int flags; 101 int ref_count; 102 int map_count; 103 bus_dma_lock_t *lockfunc; 104 void *lockfuncarg; 105 /* 106 * DMA range for this tag. If the page doesn't fall within 107 * one of these ranges, an error is returned. The caller 108 * may then decide what to do with the transfer. If the 109 * range pointer is NULL, it is ignored. 110 */ 111 struct arm32_dma_range *ranges; 112 int _nranges; 113 struct bounce_zone *bounce_zone; 114 /* 115 * Most tags need one or two segments, and can use the local tagsegs 116 * array. For tags with a larger limit, we'll allocate a bigger array 117 * on first use. 118 */ 119 bus_dma_segment_t *segments; 120 bus_dma_segment_t tagsegs[2]; 121}; 122 123struct bounce_page { 124 vm_offset_t vaddr; /* kva of bounce buffer */ 125 bus_addr_t busaddr; /* Physical address */ 126 vm_offset_t datavaddr; /* kva of client data */ 127 vm_page_t datapage; /* physical page of client data */ 128 vm_offset_t dataoffs; /* page offset of client data */ 129 bus_size_t datacount; /* client data count */ 130 STAILQ_ENTRY(bounce_page) links; 131}; 132 133struct sync_list { 134 vm_offset_t vaddr; /* kva of client data */ 135 vm_page_t pages; /* starting page of client data */ 136 vm_offset_t dataoffs; /* page offset of client data */ 137 bus_size_t datacount; /* client data count */ 138}; 139 140int busdma_swi_pending; 141 142struct bounce_zone { 143 STAILQ_ENTRY(bounce_zone) links; 144 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 145 int total_bpages; 146 int free_bpages; 147 int reserved_bpages; 148 int active_bpages; 149 int total_bounced; 150 int total_deferred; 151 int map_count; 152 bus_size_t alignment; 153 bus_addr_t lowaddr; 154 char zoneid[8]; 155 char lowaddrid[20]; 156 struct sysctl_ctx_list sysctl_tree; 157 struct sysctl_oid *sysctl_tree_top; 158}; 159 160static struct mtx bounce_lock; 161static int total_bpages; 162static int busdma_zonecount; 163static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 164 165static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 166SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 167 "Total bounce pages"); 168 169#define DMAMAP_COHERENT 0x8 170#define DMAMAP_CACHE_ALIGNED 0x10 171 172struct bus_dmamap { 173 struct bp_list bpages; 174 int pagesneeded; 175 int pagesreserved; 176 bus_dma_tag_t dmat; 177 struct memdesc mem; 178 int flags; 179 STAILQ_ENTRY(bus_dmamap) links; 180 bus_dmamap_callback_t *callback; 181 void *callback_arg; 182 int sync_count; 183 struct sync_list *slist; 184}; 185 186static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 187static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 188 189static struct mtx busdma_mtx; 190 191MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 192 193static void init_bounce_pages(void *dummy); 194static int alloc_bounce_zone(bus_dma_tag_t dmat); 195static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 196static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 197 int commit); 198static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 199 vm_offset_t vaddr, bus_addr_t addr, 200 bus_size_t size); 201static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 202static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 203 int bufaligned); 204 205/* Default tag, as most drivers provide no parent tag. */ 206bus_dma_tag_t arm_root_dma_tag; 207 208/* 209 * ---------------------------------------------------------------------------- 210 * Begin block of code useful to transplant to other implementations. 211 */ 212 213static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 214 215static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 216static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 217 218/* 219 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 220 * It'll need platform-specific changes if this code is copied. 221 */ 222static int 223dmamap_ctor(void *mem, int size, void *arg, int flags) 224{ 225 bus_dmamap_t map; 226 bus_dma_tag_t dmat; 227 228 map = (bus_dmamap_t)mem; 229 dmat = (bus_dma_tag_t)arg; 230 231 dmat->map_count++; 232 233 map->dmat = dmat; 234 map->flags = 0; 235 STAILQ_INIT(&map->bpages); 236 237 return (0); 238} 239 240/* 241 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 242 * It may need platform-specific changes if this code is copied . 243 */ 244static void 245dmamap_dtor(void *mem, int size, void *arg) 246{ 247 bus_dmamap_t map; 248 249 map = (bus_dmamap_t)mem; 250 251 map->dmat->map_count--; 252} 253 254static void 255busdma_init(void *dummy) 256{ 257 258 /* Create a cache of maps for bus_dmamap_create(). */ 259 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 260 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 261 262 /* Create a cache of buffers in standard (cacheable) memory. */ 263 standard_allocator = busdma_bufalloc_create("buffer", 264 arm_dcache_align, /* minimum_alignment */ 265 NULL, /* uma_alloc func */ 266 NULL, /* uma_free func */ 267 0); /* uma_zcreate_flags */ 268 269 /* 270 * Create a cache of buffers in uncacheable memory, to implement the 271 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 272 */ 273 coherent_allocator = busdma_bufalloc_create("coherent", 274 arm_dcache_align, /* minimum_alignment */ 275 busdma_bufalloc_alloc_uncacheable, 276 busdma_bufalloc_free_uncacheable, 277 0); /* uma_zcreate_flags */ 278} 279 280/* 281 * This init historically used SI_SUB_VM, but now the init code requires 282 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 283 * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 284 * SI_SUB_KMEM and SI_ORDER_FOURTH. 285 */ 286SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 287 288/* 289 * End block of code useful to transplant to other implementations. 290 * ---------------------------------------------------------------------------- 291 */ 292 293/* 294 * Return true if a match is made. 295 * 296 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 297 * 298 * If paddr is within the bounds of the dma tag then call the filter callback 299 * to check for a match, if there is no filter callback then assume a match. 300 */ 301static int 302run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 303{ 304 int retval; 305 306 retval = 0; 307 308 do { 309 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 310 || ((paddr & (dmat->alignment - 1)) != 0)) 311 && (dmat->filter == NULL 312 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 313 retval = 1; 314 315 dmat = dmat->parent; 316 } while (retval == 0 && dmat != NULL); 317 return (retval); 318} 319 320/* 321 * This routine checks the exclusion zone constraints from a tag against the 322 * physical RAM available on the machine. If a tag specifies an exclusion zone 323 * but there's no RAM in that zone, then we avoid allocating resources to bounce 324 * a request, and we can use any memory allocator (as opposed to needing 325 * kmem_alloc_contig() just because it can allocate pages in an address range). 326 * 327 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 328 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 329 * possibly have RAM at an address higher than the highest address we can 330 * express, so we take a fast out. 331 */ 332static __inline int 333_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 334{ 335 int i; 336 337 if (lowaddr >= BUS_SPACE_MAXADDR) 338 return (0); 339 340 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 341 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 342 || (lowaddr < phys_avail[i] && 343 highaddr > phys_avail[i])) 344 return (1); 345 } 346 return (0); 347} 348 349static __inline struct arm32_dma_range * 350_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 351 bus_addr_t curaddr) 352{ 353 struct arm32_dma_range *dr; 354 int i; 355 356 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 357 if (curaddr >= dr->dr_sysbase && 358 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 359 return (dr); 360 } 361 362 return (NULL); 363} 364/* 365 * Convenience function for manipulating driver locks from busdma (during 366 * busdma_swi, for example). Drivers that don't provide their own locks 367 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 368 * non-mutex locking scheme don't have to use this at all. 369 */ 370void 371busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 372{ 373 struct mtx *dmtx; 374 375 dmtx = (struct mtx *)arg; 376 switch (op) { 377 case BUS_DMA_LOCK: 378 mtx_lock(dmtx); 379 break; 380 case BUS_DMA_UNLOCK: 381 mtx_unlock(dmtx); 382 break; 383 default: 384 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 385 } 386} 387 388/* 389 * dflt_lock should never get called. It gets put into the dma tag when 390 * lockfunc == NULL, which is only valid if the maps that are associated 391 * with the tag are meant to never be defered. 392 * XXX Should have a way to identify which driver is responsible here. 393 */ 394static void 395dflt_lock(void *arg, bus_dma_lock_op_t op) 396{ 397#ifdef INVARIANTS 398 panic("driver error: busdma dflt_lock called"); 399#else 400 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 401#endif 402} 403 404/* 405 * Allocate a device specific dma_tag. 406 */ 407#define SEG_NB 1024 408 409int 410bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 411 bus_addr_t boundary, bus_addr_t lowaddr, 412 bus_addr_t highaddr, bus_dma_filter_t *filter, 413 void *filterarg, bus_size_t maxsize, int nsegments, 414 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 415 void *lockfuncarg, bus_dma_tag_t *dmat) 416{ 417 bus_dma_tag_t newtag; 418 int error = 0; 419 /* Return a NULL tag on failure */ 420 *dmat = NULL; 421 if (!parent) 422 parent = arm_root_dma_tag; 423 424 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 425 if (newtag == NULL) { 426 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 427 __func__, newtag, 0, error); 428 return (ENOMEM); 429 } 430 431 newtag->parent = parent; 432 newtag->alignment = alignment ? alignment : 1; 433 newtag->boundary = boundary; 434 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 435 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 436 newtag->filter = filter; 437 newtag->filterarg = filterarg; 438 newtag->maxsize = maxsize; 439 newtag->nsegments = nsegments; 440 newtag->maxsegsz = maxsegsz; 441 newtag->flags = flags; 442 newtag->ref_count = 1; /* Count ourself */ 443 newtag->map_count = 0; 444 newtag->ranges = bus_dma_get_range(); 445 newtag->_nranges = bus_dma_get_range_nb(); 446 if (lockfunc != NULL) { 447 newtag->lockfunc = lockfunc; 448 newtag->lockfuncarg = lockfuncarg; 449 } else { 450 newtag->lockfunc = dflt_lock; 451 newtag->lockfuncarg = NULL; 452 } 453 /* 454 * If all the segments we need fit into the local tagsegs array, set the 455 * pointer now. Otherwise NULL the pointer and an array of segments 456 * will be allocated later, on first use. We don't pre-allocate now 457 * because some tags exist just to pass contraints to children in the 458 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 459 * sure don't want to try to allocate an array for that. 460 */ 461 if (newtag->nsegments <= nitems(newtag->tagsegs)) 462 newtag->segments = newtag->tagsegs; 463 else 464 newtag->segments = NULL; 465 /* 466 * Take into account any restrictions imposed by our parent tag 467 */ 468 if (parent != NULL) { 469 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 470 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 471 if (newtag->boundary == 0) 472 newtag->boundary = parent->boundary; 473 else if (parent->boundary != 0) 474 newtag->boundary = MIN(parent->boundary, 475 newtag->boundary); 476 if ((newtag->filter != NULL) || 477 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 478 newtag->flags |= BUS_DMA_COULD_BOUNCE; 479 if (newtag->filter == NULL) { 480 /* 481 * Short circuit looking at our parent directly 482 * since we have encapsulated all of its information 483 */ 484 newtag->filter = parent->filter; 485 newtag->filterarg = parent->filterarg; 486 newtag->parent = parent->parent; 487 } 488 if (newtag->parent != NULL) 489 atomic_add_int(&parent->ref_count, 1); 490 } 491 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 492 || newtag->alignment > 1) 493 newtag->flags |= BUS_DMA_COULD_BOUNCE; 494 495 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 496 (flags & BUS_DMA_ALLOCNOW) != 0) { 497 struct bounce_zone *bz; 498 499 /* Must bounce */ 500 501 if ((error = alloc_bounce_zone(newtag)) != 0) { 502 free(newtag, M_DEVBUF); 503 return (error); 504 } 505 bz = newtag->bounce_zone; 506 507 if (ptoa(bz->total_bpages) < maxsize) { 508 int pages; 509 510 pages = atop(maxsize) - bz->total_bpages; 511 512 /* Add pages to our bounce pool */ 513 if (alloc_bounce_pages(newtag, pages) < pages) 514 error = ENOMEM; 515 } 516 /* Performed initial allocation */ 517 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 518 } else 519 newtag->bounce_zone = NULL; 520 if (error != 0) 521 free(newtag, M_DEVBUF); 522 else 523 *dmat = newtag; 524 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 525 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 526 527 return (error); 528} 529 530int 531bus_dma_tag_destroy(bus_dma_tag_t dmat) 532{ 533#ifdef KTR 534 bus_dma_tag_t dmat_copy = dmat; 535#endif 536 537 if (dmat != NULL) { 538 539 if (dmat->map_count != 0) 540 return (EBUSY); 541 542 while (dmat != NULL) { 543 bus_dma_tag_t parent; 544 545 parent = dmat->parent; 546 atomic_subtract_int(&dmat->ref_count, 1); 547 if (dmat->ref_count == 0) { 548 if (dmat->segments != NULL && 549 dmat->segments != dmat->tagsegs) 550 free(dmat->segments, M_DEVBUF); 551 free(dmat, M_DEVBUF); 552 /* 553 * Last reference count, so 554 * release our reference 555 * count on our parent. 556 */ 557 dmat = parent; 558 } else 559 dmat = NULL; 560 } 561 } 562 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 563 564 return (0); 565} 566 567#include <sys/kdb.h> 568/* 569 * Allocate a handle for mapping from kva/uva/physical 570 * address space into bus device space. 571 */ 572int 573bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 574{ 575 struct sync_list *slist; 576 bus_dmamap_t map; 577 int error = 0; 578 579 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 580 if (slist == NULL) 581 return (ENOMEM); 582 583 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 584 *mapp = map; 585 if (map == NULL) { 586 free(slist, M_DEVBUF); 587 return (ENOMEM); 588 } 589 590 /* 591 * If the tag's segments haven't been allocated yet we need to do it 592 * now, because we can't sleep for resources at map load time. 593 */ 594 if (dmat->segments == NULL) { 595 dmat->segments = malloc(dmat->nsegments * 596 sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 597 if (dmat->segments == NULL) { 598 free(slist, M_DEVBUF); 599 uma_zfree(dmamap_zone, map); 600 *mapp = NULL; 601 return (ENOMEM); 602 } 603 } 604 605 /* 606 * Bouncing might be required if the driver asks for an active 607 * exclusion region, a data alignment that is stricter than 1, and/or 608 * an active address boundary. 609 */ 610 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 611 612 /* Must bounce */ 613 struct bounce_zone *bz; 614 int maxpages; 615 616 if (dmat->bounce_zone == NULL) { 617 if ((error = alloc_bounce_zone(dmat)) != 0) { 618 free(slist, M_DEVBUF); 619 uma_zfree(dmamap_zone, map); 620 *mapp = NULL; 621 return (error); 622 } 623 } 624 bz = dmat->bounce_zone; 625 626 /* Initialize the new map */ 627 STAILQ_INIT(&((*mapp)->bpages)); 628 629 /* 630 * Attempt to add pages to our pool on a per-instance 631 * basis up to a sane limit. 632 */ 633 maxpages = MAX_BPAGES; 634 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 635 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 636 int pages; 637 638 pages = MAX(atop(dmat->maxsize), 1); 639 pages = MIN(maxpages - bz->total_bpages, pages); 640 pages = MAX(pages, 1); 641 if (alloc_bounce_pages(dmat, pages) < pages) 642 error = ENOMEM; 643 644 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 645 if (error == 0) 646 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 647 } else { 648 error = 0; 649 } 650 } 651 bz->map_count++; 652 } 653 map->sync_count = 0; 654 map->slist = slist; 655 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 656 __func__, dmat, dmat->flags, error); 657 658 return (0); 659} 660 661/* 662 * Destroy a handle for mapping from kva/uva/physical 663 * address space into bus device space. 664 */ 665int 666bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 667{ 668 669 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 670 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 671 __func__, dmat, EBUSY); 672 return (EBUSY); 673 } 674 free(map->slist, M_DEVBUF); 675 uma_zfree(dmamap_zone, map); 676 if (dmat->bounce_zone) 677 dmat->bounce_zone->map_count--; 678 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 679 return (0); 680} 681 682/* 683 * Allocate a piece of memory that can be efficiently mapped into bus device 684 * space based on the constraints listed in the dma tag. Returns a pointer to 685 * the allocated memory, and a pointer to an associated bus_dmamap. 686 */ 687int 688bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 689 bus_dmamap_t *mapp) 690{ 691 struct sync_list *slist; 692 void * vaddr; 693 struct busdma_bufzone *bufzone; 694 busdma_bufalloc_t ba; 695 bus_dmamap_t map; 696 int mflags; 697 vm_memattr_t memattr; 698 699 if (flags & BUS_DMA_NOWAIT) 700 mflags = M_NOWAIT; 701 else 702 mflags = M_WAITOK; 703 /* 704 * If the tag's segments haven't been allocated yet we need to do it 705 * now, because we can't sleep for resources at map load time. 706 */ 707 if (dmat->segments == NULL) 708 dmat->segments = malloc(dmat->nsegments * 709 sizeof(*dmat->segments), M_DEVBUF, mflags); 710 711 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 712 if (slist == NULL) 713 return (ENOMEM); 714 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 715 if (map == NULL) { 716 free(slist, M_DEVBUF); 717 return (ENOMEM); 718 } 719 if (flags & BUS_DMA_COHERENT) { 720 memattr = VM_MEMATTR_UNCACHEABLE; 721 ba = coherent_allocator; 722 map->flags |= DMAMAP_COHERENT; 723 } else { 724 memattr = VM_MEMATTR_DEFAULT; 725 ba = standard_allocator; 726 } 727 /* All buffers we allocate are cache-aligned. */ 728 map->flags |= DMAMAP_CACHE_ALIGNED; 729 730 if (flags & BUS_DMA_ZERO) 731 mflags |= M_ZERO; 732 733 /* 734 * Try to find a bufzone in the allocator that holds a cache of buffers 735 * of the right size for this request. If the buffer is too big to be 736 * held in the allocator cache, this returns NULL. 737 */ 738 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 739 740 /* 741 * Allocate the buffer from the uma(9) allocator if... 742 * - It's small enough to be in the allocator (bufzone not NULL). 743 * - The alignment constraint isn't larger than the allocation size 744 * (the allocator aligns buffers to their size boundaries). 745 * - There's no need to handle lowaddr/highaddr exclusion zones. 746 * else allocate non-contiguous pages if... 747 * - The page count that could get allocated doesn't exceed nsegments. 748 * - The alignment constraint isn't larger than a page boundary. 749 * - There are no boundary-crossing constraints. 750 * else allocate a block of contiguous pages because one or more of the 751 * constraints is something that only the contig allocator can fulfill. 752 */ 753 if (bufzone != NULL && dmat->alignment <= bufzone->size && 754 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 755 vaddr = uma_zalloc(bufzone->umazone, mflags); 756 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 757 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 758 vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 759 mflags, 0, dmat->lowaddr, memattr); 760 } else { 761 vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 762 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 763 memattr); 764 } 765 if (vaddr == NULL) { 766 free(slist, M_DEVBUF); 767 uma_zfree(dmamap_zone, map); 768 map = NULL; 769 } else { 770 map->slist = slist; 771 map->sync_count = 0; 772 } 773 *vaddrp = vaddr; 774 *mapp = map; 775 776 return (vaddr == NULL ? ENOMEM : 0); 777} 778 779/* 780 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 781 * its associated map. 782 */ 783void 784bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 785{ 786 struct busdma_bufzone *bufzone; 787 busdma_bufalloc_t ba; 788 789 if (map->flags & DMAMAP_COHERENT) 790 ba = coherent_allocator; 791 else 792 ba = standard_allocator; 793 794 free(map->slist, M_DEVBUF); 795 uma_zfree(dmamap_zone, map); 796 797 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 798 799 if (bufzone != NULL && dmat->alignment <= bufzone->size && 800 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 801 uma_zfree(bufzone->umazone, vaddr); 802 else 803 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 804} 805 806static void 807_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 808 bus_size_t buflen, int flags) 809{ 810 bus_addr_t curaddr; 811 bus_size_t sgsize; 812 813 if (map->pagesneeded == 0) { 814 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 815 dmat->lowaddr, dmat->boundary, dmat->alignment); 816 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 817 map, map->pagesneeded); 818 /* 819 * Count the number of bounce pages 820 * needed in order to complete this transfer 821 */ 822 curaddr = buf; 823 while (buflen != 0) { 824 sgsize = MIN(buflen, dmat->maxsegsz); 825 if (run_filter(dmat, curaddr) != 0) { 826 sgsize = MIN(sgsize, 827 PAGE_SIZE - (curaddr & PAGE_MASK)); 828 map->pagesneeded++; 829 } 830 curaddr += sgsize; 831 buflen -= sgsize; 832 } 833 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 834 } 835} 836 837static void 838_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 839 void *buf, bus_size_t buflen, int flags) 840{ 841 vm_offset_t vaddr; 842 vm_offset_t vendaddr; 843 bus_addr_t paddr; 844 845 if (map->pagesneeded == 0) { 846 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 847 dmat->lowaddr, dmat->boundary, dmat->alignment); 848 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 849 map, map->pagesneeded); 850 /* 851 * Count the number of bounce pages 852 * needed in order to complete this transfer 853 */ 854 vaddr = trunc_page((vm_offset_t)buf); 855 vendaddr = (vm_offset_t)buf + buflen; 856 857 while (vaddr < vendaddr) { 858 if (__predict_true(pmap == kernel_pmap)) 859 paddr = pmap_kextract(vaddr); 860 else 861 paddr = pmap_extract(pmap, vaddr); 862 if (run_filter(dmat, paddr) != 0) 863 map->pagesneeded++; 864 vaddr += PAGE_SIZE; 865 } 866 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 867 } 868} 869 870static int 871_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 872{ 873 874 /* Reserve Necessary Bounce Pages */ 875 mtx_lock(&bounce_lock); 876 if (flags & BUS_DMA_NOWAIT) { 877 if (reserve_bounce_pages(dmat, map, 0) != 0) { 878 mtx_unlock(&bounce_lock); 879 return (ENOMEM); 880 } 881 } else { 882 if (reserve_bounce_pages(dmat, map, 1) != 0) { 883 /* Queue us for resources */ 884 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 885 mtx_unlock(&bounce_lock); 886 return (EINPROGRESS); 887 } 888 } 889 mtx_unlock(&bounce_lock); 890 891 return (0); 892} 893 894/* 895 * Add a single contiguous physical range to the segment list. 896 */ 897static int 898_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 899 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 900{ 901 bus_addr_t baddr, bmask; 902 int seg; 903 904 /* 905 * Make sure we don't cross any boundaries. 906 */ 907 bmask = ~(dmat->boundary - 1); 908 if (dmat->boundary > 0) { 909 baddr = (curaddr + dmat->boundary) & bmask; 910 if (sgsize > (baddr - curaddr)) 911 sgsize = (baddr - curaddr); 912 } 913 if (dmat->ranges) { 914 struct arm32_dma_range *dr; 915 916 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 917 curaddr); 918 if (dr == NULL) 919 return (0); 920 /* 921 * In a valid DMA range. Translate the physical 922 * memory address to an address in the DMA window. 923 */ 924 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 925 926 } 927 928 seg = *segp; 929 /* 930 * Insert chunk into a segment, coalescing with 931 * the previous segment if possible. 932 */ 933 if (seg >= 0 && 934 curaddr == segs[seg].ds_addr + segs[seg].ds_len && 935 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 936 (dmat->boundary == 0 || 937 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 938 segs[seg].ds_len += sgsize; 939 } else { 940 if (++seg >= dmat->nsegments) 941 return (0); 942 segs[seg].ds_addr = curaddr; 943 segs[seg].ds_len = sgsize; 944 } 945 *segp = seg; 946 return (sgsize); 947} 948 949/* 950 * Utility function to load a physical buffer. segp contains 951 * the starting segment on entrace, and the ending segment on exit. 952 */ 953int 954_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 955 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 956{ 957 struct sync_list *sl; 958 bus_size_t sgsize; 959 bus_addr_t curaddr; 960 bus_addr_t sl_end = 0; 961 int error; 962 963 if (segs == NULL) 964 segs = dmat->segments; 965 966 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 967 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 968 if (map->pagesneeded != 0) { 969 error = _bus_dmamap_reserve_pages(dmat, map, flags); 970 if (error) 971 return (error); 972 } 973 } 974 975 sl = map->slist + map->sync_count - 1; 976 977 while (buflen > 0) { 978 curaddr = buf; 979 sgsize = MIN(buflen, dmat->maxsegsz); 980 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 981 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 982 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 983 curaddr = add_bounce_page(dmat, map, 0, curaddr, 984 sgsize); 985 } else { 986 if (map->sync_count > 0) 987 sl_end = VM_PAGE_TO_PHYS(sl->pages) + 988 sl->dataoffs + sl->datacount; 989 990 if (map->sync_count == 0 || curaddr != sl_end) { 991 if (++map->sync_count > dmat->nsegments) 992 break; 993 sl++; 994 sl->vaddr = 0; 995 sl->datacount = sgsize; 996 /* 997 * PHYS_TO_VM_PAGE() will truncate 998 * unaligned addresses. 999 */ 1000 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1001 sl->dataoffs = curaddr & PAGE_MASK; 1002 } else 1003 sl->datacount += sgsize; 1004 } 1005 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1006 segp); 1007 if (sgsize == 0) 1008 break; 1009 buf += sgsize; 1010 buflen -= sgsize; 1011 } 1012 1013 /* 1014 * Did we fit? 1015 */ 1016 if (buflen != 0) { 1017 _bus_dmamap_unload(dmat, map); 1018 return (EFBIG); /* XXX better return value here? */ 1019 } 1020 return (0); 1021} 1022 1023int 1024_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1025 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1026 bus_dma_segment_t *segs, int *segp) 1027{ 1028 1029 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1030 segs, segp)); 1031} 1032 1033/* 1034 * Utility function to load a linear buffer. segp contains 1035 * the starting segment on entrance, and the ending segment on exit. 1036 */ 1037int 1038_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1039 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1040 int *segp) 1041{ 1042 bus_size_t sgsize; 1043 bus_addr_t curaddr; 1044 bus_addr_t sl_pend = 0; 1045 struct sync_list *sl; 1046 vm_offset_t kvaddr; 1047 vm_offset_t vaddr = (vm_offset_t)buf; 1048 vm_offset_t sl_vend = 0; 1049 int error = 0; 1050 1051 if (segs == NULL) 1052 segs = dmat->segments; 1053 if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1054 map->flags |= DMAMAP_CACHE_ALIGNED; 1055 1056 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1057 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1058 if (map->pagesneeded != 0) { 1059 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1060 if (error) 1061 return (error); 1062 } 1063 } 1064 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1065 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1066 1067 sl = map->slist + map->sync_count - 1; 1068 1069 while (buflen > 0) { 1070 /* 1071 * Get the physical address for this segment. 1072 */ 1073 if (__predict_true(pmap == kernel_pmap)) { 1074 curaddr = pmap_kextract(vaddr); 1075 kvaddr = vaddr; 1076 } else { 1077 curaddr = pmap_extract(pmap, vaddr); 1078 map->flags &= ~DMAMAP_COHERENT; 1079 kvaddr = 0; 1080 } 1081 1082 /* 1083 * Compute the segment size, and adjust counts. 1084 */ 1085 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1086 if (sgsize > dmat->maxsegsz) 1087 sgsize = dmat->maxsegsz; 1088 if (buflen < sgsize) 1089 sgsize = buflen; 1090 1091 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1092 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1093 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1094 sgsize); 1095 } else { 1096 if (map->sync_count > 0) { 1097 sl_pend = VM_PAGE_TO_PHYS(sl->pages) + 1098 sl->dataoffs + sl->datacount; 1099 sl_vend = sl->vaddr + sl->datacount; 1100 } 1101 1102 if (map->sync_count == 0 || 1103 (kvaddr != 0 && kvaddr != sl_vend) || 1104 (kvaddr == 0 && curaddr != sl_pend)) { 1105 1106 if (++map->sync_count > dmat->nsegments) 1107 goto cleanup; 1108 sl++; 1109 sl->vaddr = kvaddr; 1110 sl->datacount = sgsize; 1111 /* 1112 * PHYS_TO_VM_PAGE() will truncate 1113 * unaligned addresses. 1114 */ 1115 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1116 sl->dataoffs = curaddr & PAGE_MASK; 1117 } else 1118 sl->datacount += sgsize; 1119 } 1120 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1121 segp); 1122 if (sgsize == 0) 1123 break; 1124 vaddr += sgsize; 1125 buflen -= sgsize; 1126 } 1127 1128cleanup: 1129 /* 1130 * Did we fit? 1131 */ 1132 if (buflen != 0) { 1133 _bus_dmamap_unload(dmat, map); 1134 return (EFBIG); /* XXX better return value here? */ 1135 } 1136 return (0); 1137} 1138 1139void 1140__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 1141 struct memdesc *mem, bus_dmamap_callback_t *callback, 1142 void *callback_arg) 1143{ 1144 1145 KASSERT(dmat != NULL, ("dmatag is NULL")); 1146 KASSERT(map != NULL, ("dmamap is NULL")); 1147 map->mem = *mem; 1148 map->callback = callback; 1149 map->callback_arg = callback_arg; 1150} 1151 1152bus_dma_segment_t * 1153_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1154 bus_dma_segment_t *segs, int nsegs, int error) 1155{ 1156 1157 if (segs == NULL) 1158 segs = dmat->segments; 1159 return (segs); 1160} 1161 1162/* 1163 * Release the mapping held by map. 1164 */ 1165void 1166_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1167{ 1168 struct bounce_page *bpage; 1169 1170 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1171 STAILQ_REMOVE_HEAD(&map->bpages, links); 1172 free_bounce_page(dmat, bpage); 1173 } 1174 map->sync_count = 0; 1175 return; 1176} 1177 1178static void 1179bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1180 int bufaligned) 1181{ 1182 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1183 register_t s; 1184 int partial; 1185 1186 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1187 cpu_dcache_wb_range(buf, len); 1188 cpu_l2cache_wb_range(buf, len); 1189 } 1190 1191 /* 1192 * If the caller promises the buffer is properly aligned to a cache line 1193 * (even if the call parms make it look like it isn't) we can avoid 1194 * attempting to preserve the non-DMA part of the cache line in the 1195 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1196 * 1197 * This covers the case of mbufs, where we know how they're aligned and 1198 * know the CPU doesn't touch the header in front of the DMA data area 1199 * during the IO, but it may have touched it right before invoking the 1200 * sync, so a PREREAD writeback is required. 1201 * 1202 * It also handles buffers we created in bus_dmamem_alloc(), which are 1203 * always aligned and padded to cache line size even if the IO length 1204 * isn't a multiple of cache line size. In this case the PREREAD 1205 * writeback probably isn't required, but it's harmless. 1206 */ 1207 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1208 1209 if (op & BUS_DMASYNC_PREREAD) { 1210 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1211 cpu_dcache_inv_range(buf, len); 1212 cpu_l2cache_inv_range(buf, len); 1213 } else { 1214 cpu_dcache_wbinv_range(buf, len); 1215 cpu_l2cache_wbinv_range(buf, len); 1216 } 1217 } 1218 if (op & BUS_DMASYNC_POSTREAD) { 1219 if (partial && !bufaligned) { 1220 s = intr_disable(); 1221 if (buf & arm_dcache_align_mask) 1222 memcpy(_tmp_cl, (void *)(buf & 1223 ~arm_dcache_align_mask), 1224 buf & arm_dcache_align_mask); 1225 if ((buf + len) & arm_dcache_align_mask) 1226 memcpy(_tmp_clend, 1227 (void *)(buf + len), 1228 arm_dcache_align - 1229 ((buf + len) & arm_dcache_align_mask)); 1230 } 1231 cpu_dcache_inv_range(buf, len); 1232 cpu_l2cache_inv_range(buf, len); 1233 if (partial && !bufaligned) { 1234 if (buf & arm_dcache_align_mask) 1235 memcpy((void *)(buf & 1236 ~arm_dcache_align_mask), _tmp_cl, 1237 buf & arm_dcache_align_mask); 1238 if ((buf + len) & arm_dcache_align_mask) 1239 memcpy((void *)(buf + len), 1240 _tmp_clend, arm_dcache_align - 1241 ((buf + len) & arm_dcache_align_mask)); 1242 intr_restore(s); 1243 } 1244 } 1245} 1246 1247static void 1248bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 1249 int bufaligned) 1250{ 1251 vm_offset_t tempvaddr; 1252 vm_page_t curpage; 1253 size_t npages; 1254 1255 if (sl->vaddr != 0) { 1256 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); 1257 return; 1258 } 1259 1260 tempvaddr = 0; 1261 npages = atop(round_page(sl->dataoffs + sl->datacount)); 1262 1263 for (curpage = sl->pages; curpage != sl->pages + npages; ++curpage) { 1264 /* 1265 * If the page is mapped to some other VA that hasn't 1266 * been supplied to busdma, then pmap_quick_enter_page() 1267 * will find all duplicate mappings and mark them 1268 * uncacheable. 1269 * That will also do any necessary wb/inv. Otherwise, 1270 * if the page is truly unmapped, then we don't actually 1271 * need to do cache maintenance. 1272 * XXX: May overwrite DMA'ed data in the POSTREAD 1273 * case where the CPU has written to a cacheline not 1274 * completely covered by the DMA region. 1275 */ 1276 KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) + 1277 ptoa(curpage - sl->pages), 1278 ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1279 VM_PAGE_TO_PHYS(curpage), VM_PAGE_TO_PHYS(sl->pages) + 1280 ptoa(curpage - sl->pages))); 1281 tempvaddr = pmap_quick_enter_page(curpage); 1282 pmap_quick_remove_page(tempvaddr); 1283 } 1284} 1285 1286static void 1287_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1288{ 1289 struct bounce_page *bpage; 1290 vm_offset_t datavaddr, tempvaddr; 1291 1292 if ((op & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) == 0) 1293 return; 1294 1295 STAILQ_FOREACH(bpage, &map->bpages, links) { 1296 tempvaddr = 0; 1297 datavaddr = bpage->datavaddr; 1298 if (op & BUS_DMASYNC_PREWRITE) { 1299 if (datavaddr == 0) { 1300 tempvaddr = 1301 pmap_quick_enter_page(bpage->datapage); 1302 datavaddr = tempvaddr | bpage->dataoffs; 1303 } 1304 bcopy((void *)datavaddr, 1305 (void *)bpage->vaddr, bpage->datacount); 1306 if (tempvaddr != 0) 1307 pmap_quick_remove_page(tempvaddr); 1308 cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1309 cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1310 dmat->bounce_zone->total_bounced++; 1311 } 1312 if (op & BUS_DMASYNC_POSTREAD) { 1313 cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1314 cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1315 if (datavaddr == 0) { 1316 tempvaddr = 1317 pmap_quick_enter_page(bpage->datapage); 1318 datavaddr = tempvaddr | bpage->dataoffs; 1319 } 1320 bcopy((void *)bpage->vaddr, 1321 (void *)datavaddr, bpage->datacount); 1322 if (tempvaddr != 0) 1323 pmap_quick_remove_page(tempvaddr); 1324 dmat->bounce_zone->total_bounced++; 1325 } 1326 } 1327} 1328 1329void 1330_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1331{ 1332 struct sync_list *sl, *end; 1333 int bufaligned; 1334 1335 if (op == BUS_DMASYNC_POSTWRITE) 1336 return; 1337 if (map->flags & DMAMAP_COHERENT) 1338 goto drain; 1339 if (STAILQ_FIRST(&map->bpages)) 1340 _bus_dmamap_sync_bp(dmat, map, op); 1341 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1342 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1343 if (map->sync_count) { 1344 end = &map->slist[map->sync_count]; 1345 for (sl = &map->slist[0]; sl != end; sl++) 1346 bus_dmamap_sync_sl(sl, op, bufaligned); 1347 } 1348 1349drain: 1350 1351 cpu_drain_writebuf(); 1352} 1353 1354static void 1355init_bounce_pages(void *dummy __unused) 1356{ 1357 1358 total_bpages = 0; 1359 STAILQ_INIT(&bounce_zone_list); 1360 STAILQ_INIT(&bounce_map_waitinglist); 1361 STAILQ_INIT(&bounce_map_callbacklist); 1362 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1363} 1364SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1365 1366static struct sysctl_ctx_list * 1367busdma_sysctl_tree(struct bounce_zone *bz) 1368{ 1369 return (&bz->sysctl_tree); 1370} 1371 1372static struct sysctl_oid * 1373busdma_sysctl_tree_top(struct bounce_zone *bz) 1374{ 1375 return (bz->sysctl_tree_top); 1376} 1377 1378static int 1379alloc_bounce_zone(bus_dma_tag_t dmat) 1380{ 1381 struct bounce_zone *bz; 1382 1383 /* Check to see if we already have a suitable zone */ 1384 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1385 if ((dmat->alignment <= bz->alignment) 1386 && (dmat->lowaddr >= bz->lowaddr)) { 1387 dmat->bounce_zone = bz; 1388 return (0); 1389 } 1390 } 1391 1392 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1393 M_NOWAIT | M_ZERO)) == NULL) 1394 return (ENOMEM); 1395 1396 STAILQ_INIT(&bz->bounce_page_list); 1397 bz->free_bpages = 0; 1398 bz->reserved_bpages = 0; 1399 bz->active_bpages = 0; 1400 bz->lowaddr = dmat->lowaddr; 1401 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1402 bz->map_count = 0; 1403 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1404 busdma_zonecount++; 1405 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1406 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1407 dmat->bounce_zone = bz; 1408 1409 sysctl_ctx_init(&bz->sysctl_tree); 1410 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1411 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1412 CTLFLAG_RD, 0, ""); 1413 if (bz->sysctl_tree_top == NULL) { 1414 sysctl_ctx_free(&bz->sysctl_tree); 1415 return (0); /* XXX error code? */ 1416 } 1417 1418 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1419 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1420 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1421 "Total bounce pages"); 1422 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1423 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1424 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1425 "Free bounce pages"); 1426 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1427 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1428 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1429 "Reserved bounce pages"); 1430 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1431 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1432 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1433 "Active bounce pages"); 1434 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1435 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1436 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1437 "Total bounce requests"); 1438 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1439 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1440 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1441 "Total bounce requests that were deferred"); 1442 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1443 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1444 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1445 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1446 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1447 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1448 1449 return (0); 1450} 1451 1452static int 1453alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1454{ 1455 struct bounce_zone *bz; 1456 int count; 1457 1458 bz = dmat->bounce_zone; 1459 count = 0; 1460 while (numpages > 0) { 1461 struct bounce_page *bpage; 1462 1463 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1464 M_NOWAIT | M_ZERO); 1465 1466 if (bpage == NULL) 1467 break; 1468 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1469 M_NOWAIT, 0ul, 1470 bz->lowaddr, 1471 PAGE_SIZE, 1472 0); 1473 if (bpage->vaddr == 0) { 1474 free(bpage, M_DEVBUF); 1475 break; 1476 } 1477 bpage->busaddr = pmap_kextract(bpage->vaddr); 1478 mtx_lock(&bounce_lock); 1479 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1480 total_bpages++; 1481 bz->total_bpages++; 1482 bz->free_bpages++; 1483 mtx_unlock(&bounce_lock); 1484 count++; 1485 numpages--; 1486 } 1487 return (count); 1488} 1489 1490static int 1491reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1492{ 1493 struct bounce_zone *bz; 1494 int pages; 1495 1496 mtx_assert(&bounce_lock, MA_OWNED); 1497 bz = dmat->bounce_zone; 1498 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1499 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1500 return (map->pagesneeded - (map->pagesreserved + pages)); 1501 bz->free_bpages -= pages; 1502 bz->reserved_bpages += pages; 1503 map->pagesreserved += pages; 1504 pages = map->pagesneeded - map->pagesreserved; 1505 1506 return (pages); 1507} 1508 1509static bus_addr_t 1510add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1511 bus_addr_t addr, bus_size_t size) 1512{ 1513 struct bounce_zone *bz; 1514 struct bounce_page *bpage; 1515 1516 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1517 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1518 1519 bz = dmat->bounce_zone; 1520 if (map->pagesneeded == 0) 1521 panic("add_bounce_page: map doesn't need any pages"); 1522 map->pagesneeded--; 1523 1524 if (map->pagesreserved == 0) 1525 panic("add_bounce_page: map doesn't need any pages"); 1526 map->pagesreserved--; 1527 1528 mtx_lock(&bounce_lock); 1529 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1530 if (bpage == NULL) 1531 panic("add_bounce_page: free page list is empty"); 1532 1533 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1534 bz->reserved_bpages--; 1535 bz->active_bpages++; 1536 mtx_unlock(&bounce_lock); 1537 1538 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1539 /* Page offset needs to be preserved. */ 1540 bpage->vaddr |= addr & PAGE_MASK; 1541 bpage->busaddr |= addr & PAGE_MASK; 1542 } 1543 bpage->datavaddr = vaddr; 1544 /* PHYS_TO_VM_PAGE() will truncate unaligned addresses. */ 1545 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1546 bpage->dataoffs = addr & PAGE_MASK; 1547 bpage->datacount = size; 1548 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1549 return (bpage->busaddr); 1550} 1551 1552static void 1553free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1554{ 1555 struct bus_dmamap *map; 1556 struct bounce_zone *bz; 1557 1558 bz = dmat->bounce_zone; 1559 bpage->datavaddr = 0; 1560 bpage->datacount = 0; 1561 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1562 /* 1563 * Reset the bounce page to start at offset 0. Other uses 1564 * of this bounce page may need to store a full page of 1565 * data and/or assume it starts on a page boundary. 1566 */ 1567 bpage->vaddr &= ~PAGE_MASK; 1568 bpage->busaddr &= ~PAGE_MASK; 1569 } 1570 1571 mtx_lock(&bounce_lock); 1572 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1573 bz->free_bpages++; 1574 bz->active_bpages--; 1575 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1576 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1577 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1578 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1579 map, links); 1580 busdma_swi_pending = 1; 1581 bz->total_deferred++; 1582 swi_sched(vm_ih, 0); 1583 } 1584 } 1585 mtx_unlock(&bounce_lock); 1586} 1587 1588void 1589busdma_swi(void) 1590{ 1591 bus_dma_tag_t dmat; 1592 struct bus_dmamap *map; 1593 1594 mtx_lock(&bounce_lock); 1595 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1596 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1597 mtx_unlock(&bounce_lock); 1598 dmat = map->dmat; 1599 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1600 bus_dmamap_load_mem(map->dmat, map, &map->mem, 1601 map->callback, map->callback_arg, BUS_DMA_WAITOK); 1602 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1603 mtx_lock(&bounce_lock); 1604 } 1605 mtx_unlock(&bounce_lock); 1606} 1607