busdma_machdep-v4.c revision 289857
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289857 2015-10-23 22:51:48Z ian $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/interrupt.h> 62#include <sys/lock.h> 63#include <sys/proc.h> 64#include <sys/memdesc.h> 65#include <sys/mutex.h> 66#include <sys/ktr.h> 67#include <sys/kernel.h> 68#include <sys/sysctl.h> 69#include <sys/uio.h> 70 71#include <vm/uma.h> 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74#include <vm/vm_kern.h> 75#include <vm/vm_page.h> 76#include <vm/vm_map.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86 87struct bounce_zone; 88 89struct bus_dma_tag { 90 bus_dma_tag_t parent; 91 bus_size_t alignment; 92 bus_addr_t boundary; 93 bus_addr_t lowaddr; 94 bus_addr_t highaddr; 95 bus_dma_filter_t *filter; 96 void *filterarg; 97 bus_size_t maxsize; 98 u_int nsegments; 99 bus_size_t maxsegsz; 100 int flags; 101 int ref_count; 102 int map_count; 103 bus_dma_lock_t *lockfunc; 104 void *lockfuncarg; 105 struct bounce_zone *bounce_zone; 106 /* 107 * DMA range for this tag. If the page doesn't fall within 108 * one of these ranges, an error is returned. The caller 109 * may then decide what to do with the transfer. If the 110 * range pointer is NULL, it is ignored. 111 */ 112 struct arm32_dma_range *ranges; 113 int _nranges; 114 /* 115 * Most tags need one or two segments, and can use the local tagsegs 116 * array. For tags with a larger limit, we'll allocate a bigger array 117 * on first use. 118 */ 119 bus_dma_segment_t *segments; 120 bus_dma_segment_t tagsegs[2]; 121}; 122 123struct bounce_page { 124 vm_offset_t vaddr; /* kva of bounce buffer */ 125 bus_addr_t busaddr; /* Physical address */ 126 vm_offset_t datavaddr; /* kva of client data */ 127 vm_page_t datapage; /* physical page of client data */ 128 vm_offset_t dataoffs; /* page offset of client data */ 129 bus_size_t datacount; /* client data count */ 130 STAILQ_ENTRY(bounce_page) links; 131}; 132 133struct sync_list { 134 vm_offset_t vaddr; /* kva of client data */ 135 vm_page_t pages; /* starting page of client data */ 136 vm_offset_t dataoffs; /* page offset of client data */ 137 bus_size_t datacount; /* client data count */ 138}; 139 140int busdma_swi_pending; 141 142struct bounce_zone { 143 STAILQ_ENTRY(bounce_zone) links; 144 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 145 int total_bpages; 146 int free_bpages; 147 int reserved_bpages; 148 int active_bpages; 149 int total_bounced; 150 int total_deferred; 151 int map_count; 152 bus_size_t alignment; 153 bus_addr_t lowaddr; 154 char zoneid[8]; 155 char lowaddrid[20]; 156 struct sysctl_ctx_list sysctl_tree; 157 struct sysctl_oid *sysctl_tree_top; 158}; 159 160static struct mtx bounce_lock; 161static int total_bpages; 162static int busdma_zonecount; 163static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 164 165static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 166SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 167 "Total bounce pages"); 168 169struct bus_dmamap { 170 struct bp_list bpages; 171 int pagesneeded; 172 int pagesreserved; 173 bus_dma_tag_t dmat; 174 struct memdesc mem; 175 bus_dmamap_callback_t *callback; 176 void *callback_arg; 177 int flags; 178#define DMAMAP_COHERENT 0x8 179#define DMAMAP_CACHE_ALIGNED 0x10 180 STAILQ_ENTRY(bus_dmamap) links; 181 int sync_count; 182 struct sync_list *slist; 183}; 184 185static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 186static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 187 188static struct mtx busdma_mtx; 189 190MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 191 192static void init_bounce_pages(void *dummy); 193static int alloc_bounce_zone(bus_dma_tag_t dmat); 194static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 195static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 196 int commit); 197static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 198 vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 199static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 201 int bufaligned); 202 203/* Default tag, as most drivers provide no parent tag. */ 204bus_dma_tag_t arm_root_dma_tag; 205 206/* 207 * ---------------------------------------------------------------------------- 208 * Begin block of code useful to transplant to other implementations. 209 */ 210 211static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 212 213static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 214static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 215 216MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 217MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages"); 218 219/* 220 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 221 * It'll need platform-specific changes if this code is copied. 222 */ 223static int 224dmamap_ctor(void *mem, int size, void *arg, int flags) 225{ 226 bus_dmamap_t map; 227 bus_dma_tag_t dmat; 228 229 map = (bus_dmamap_t)mem; 230 dmat = (bus_dma_tag_t)arg; 231 232 dmat->map_count++; 233 234 map->dmat = dmat; 235 map->flags = 0; 236 STAILQ_INIT(&map->bpages); 237 238 return (0); 239} 240 241/* 242 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 243 * It may need platform-specific changes if this code is copied . 244 */ 245static void 246dmamap_dtor(void *mem, int size, void *arg) 247{ 248 bus_dmamap_t map; 249 250 map = (bus_dmamap_t)mem; 251 252 map->dmat->map_count--; 253} 254 255static void 256busdma_init(void *dummy) 257{ 258 259 /* Create a cache of maps for bus_dmamap_create(). */ 260 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 261 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 262 263 /* Create a cache of buffers in standard (cacheable) memory. */ 264 standard_allocator = busdma_bufalloc_create("buffer", 265 arm_dcache_align, /* minimum_alignment */ 266 NULL, /* uma_alloc func */ 267 NULL, /* uma_free func */ 268 0); /* uma_zcreate_flags */ 269 270 /* 271 * Create a cache of buffers in uncacheable memory, to implement the 272 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 273 */ 274 coherent_allocator = busdma_bufalloc_create("coherent", 275 arm_dcache_align, /* minimum_alignment */ 276 busdma_bufalloc_alloc_uncacheable, 277 busdma_bufalloc_free_uncacheable, 278 0); /* uma_zcreate_flags */ 279} 280 281/* 282 * This init historically used SI_SUB_VM, but now the init code requires 283 * malloc(9) using M_BUSDMA memory, which is set up later than SI_SUB_VM, by 284 * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 285 * SI_SUB_KMEM and SI_ORDER_FOURTH. 286 */ 287SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 288 289/* 290 * End block of code useful to transplant to other implementations. 291 * ---------------------------------------------------------------------------- 292 */ 293 294/* 295 * Return true if a match is made. 296 * 297 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 298 * 299 * If paddr is within the bounds of the dma tag then call the filter callback 300 * to check for a match, if there is no filter callback then assume a match. 301 */ 302static int 303run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 304{ 305 int retval; 306 307 retval = 0; 308 309 do { 310 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 311 || ((paddr & (dmat->alignment - 1)) != 0)) 312 && (dmat->filter == NULL 313 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 314 retval = 1; 315 316 dmat = dmat->parent; 317 } while (retval == 0 && dmat != NULL); 318 return (retval); 319} 320 321/* 322 * This routine checks the exclusion zone constraints from a tag against the 323 * physical RAM available on the machine. If a tag specifies an exclusion zone 324 * but there's no RAM in that zone, then we avoid allocating resources to bounce 325 * a request, and we can use any memory allocator (as opposed to needing 326 * kmem_alloc_contig() just because it can allocate pages in an address range). 327 * 328 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 329 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 330 * possibly have RAM at an address higher than the highest address we can 331 * express, so we take a fast out. 332 */ 333static __inline int 334_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 335{ 336 int i; 337 338 if (lowaddr >= BUS_SPACE_MAXADDR) 339 return (0); 340 341 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 342 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 343 || (lowaddr < phys_avail[i] && 344 highaddr > phys_avail[i])) 345 return (1); 346 } 347 return (0); 348} 349 350static __inline struct arm32_dma_range * 351_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 352 bus_addr_t curaddr) 353{ 354 struct arm32_dma_range *dr; 355 int i; 356 357 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 358 if (curaddr >= dr->dr_sysbase && 359 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 360 return (dr); 361 } 362 363 return (NULL); 364} 365 366/* 367 * Convenience function for manipulating driver locks from busdma (during 368 * busdma_swi, for example). Drivers that don't provide their own locks 369 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 370 * non-mutex locking scheme don't have to use this at all. 371 */ 372void 373busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 374{ 375 struct mtx *dmtx; 376 377 dmtx = (struct mtx *)arg; 378 switch (op) { 379 case BUS_DMA_LOCK: 380 mtx_lock(dmtx); 381 break; 382 case BUS_DMA_UNLOCK: 383 mtx_unlock(dmtx); 384 break; 385 default: 386 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 387 } 388} 389 390/* 391 * dflt_lock should never get called. It gets put into the dma tag when 392 * lockfunc == NULL, which is only valid if the maps that are associated 393 * with the tag are meant to never be defered. 394 * XXX Should have a way to identify which driver is responsible here. 395 */ 396static void 397dflt_lock(void *arg, bus_dma_lock_op_t op) 398{ 399#ifdef INVARIANTS 400 panic("driver error: busdma dflt_lock called"); 401#else 402 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 403#endif 404} 405 406/* 407 * Allocate a device specific dma_tag. 408 */ 409#define SEG_NB 1024 410 411int 412bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 413 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 414 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 415 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 416 void *lockfuncarg, bus_dma_tag_t *dmat) 417{ 418 bus_dma_tag_t newtag; 419 int error = 0; 420 /* Return a NULL tag on failure */ 421 *dmat = NULL; 422 if (!parent) 423 parent = arm_root_dma_tag; 424 425 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, M_NOWAIT); 426 if (newtag == NULL) { 427 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 428 __func__, newtag, 0, error); 429 return (ENOMEM); 430 } 431 432 newtag->parent = parent; 433 newtag->alignment = alignment ? alignment : 1; 434 newtag->boundary = boundary; 435 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 436 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 437 newtag->filter = filter; 438 newtag->filterarg = filterarg; 439 newtag->maxsize = maxsize; 440 newtag->nsegments = nsegments; 441 newtag->maxsegsz = maxsegsz; 442 newtag->flags = flags; 443 newtag->ref_count = 1; /* Count ourself */ 444 newtag->map_count = 0; 445 newtag->ranges = bus_dma_get_range(); 446 newtag->_nranges = bus_dma_get_range_nb(); 447 if (lockfunc != NULL) { 448 newtag->lockfunc = lockfunc; 449 newtag->lockfuncarg = lockfuncarg; 450 } else { 451 newtag->lockfunc = dflt_lock; 452 newtag->lockfuncarg = NULL; 453 } 454 /* 455 * If all the segments we need fit into the local tagsegs array, set the 456 * pointer now. Otherwise NULL the pointer and an array of segments 457 * will be allocated later, on first use. We don't pre-allocate now 458 * because some tags exist just to pass contraints to children in the 459 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 460 * sure don't want to try to allocate an array for that. 461 */ 462 if (newtag->nsegments <= nitems(newtag->tagsegs)) 463 newtag->segments = newtag->tagsegs; 464 else 465 newtag->segments = NULL; 466 /* 467 * Take into account any restrictions imposed by our parent tag 468 */ 469 if (parent != NULL) { 470 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 471 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 472 if (newtag->boundary == 0) 473 newtag->boundary = parent->boundary; 474 else if (parent->boundary != 0) 475 newtag->boundary = MIN(parent->boundary, 476 newtag->boundary); 477 if ((newtag->filter != NULL) || 478 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 479 newtag->flags |= BUS_DMA_COULD_BOUNCE; 480 if (newtag->filter == NULL) { 481 /* 482 * Short circuit looking at our parent directly 483 * since we have encapsulated all of its information 484 */ 485 newtag->filter = parent->filter; 486 newtag->filterarg = parent->filterarg; 487 newtag->parent = parent->parent; 488 } 489 if (newtag->parent != NULL) 490 atomic_add_int(&parent->ref_count, 1); 491 } 492 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 493 || newtag->alignment > 1) 494 newtag->flags |= BUS_DMA_COULD_BOUNCE; 495 496 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 497 (flags & BUS_DMA_ALLOCNOW) != 0) { 498 struct bounce_zone *bz; 499 500 /* Must bounce */ 501 502 if ((error = alloc_bounce_zone(newtag)) != 0) { 503 free(newtag, M_BUSDMA); 504 return (error); 505 } 506 bz = newtag->bounce_zone; 507 508 if (ptoa(bz->total_bpages) < maxsize) { 509 int pages; 510 511 pages = atop(maxsize) - bz->total_bpages; 512 513 /* Add pages to our bounce pool */ 514 if (alloc_bounce_pages(newtag, pages) < pages) 515 error = ENOMEM; 516 } 517 /* Performed initial allocation */ 518 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 519 } else 520 newtag->bounce_zone = NULL; 521 if (error != 0) 522 free(newtag, M_BUSDMA); 523 else 524 *dmat = newtag; 525 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 526 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 527 528 return (error); 529} 530 531int 532bus_dma_tag_destroy(bus_dma_tag_t dmat) 533{ 534#ifdef KTR 535 bus_dma_tag_t dmat_copy = dmat; 536#endif 537 538 if (dmat != NULL) { 539 540 if (dmat->map_count != 0) 541 return (EBUSY); 542 543 while (dmat != NULL) { 544 bus_dma_tag_t parent; 545 546 parent = dmat->parent; 547 atomic_subtract_int(&dmat->ref_count, 1); 548 if (dmat->ref_count == 0) { 549 if (dmat->segments != NULL && 550 dmat->segments != dmat->tagsegs) 551 free(dmat->segments, M_BUSDMA); 552 free(dmat, M_BUSDMA); 553 /* 554 * Last reference count, so 555 * release our reference 556 * count on our parent. 557 */ 558 dmat = parent; 559 } else 560 dmat = NULL; 561 } 562 } 563 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 564 565 return (0); 566} 567 568#include <sys/kdb.h> 569/* 570 * Allocate a handle for mapping from kva/uva/physical 571 * address space into bus device space. 572 */ 573int 574bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 575{ 576 struct sync_list *slist; 577 bus_dmamap_t map; 578 int error = 0; 579 580 slist = malloc(sizeof(*slist) * dmat->nsegments, M_BUSDMA, M_NOWAIT); 581 if (slist == NULL) 582 return (ENOMEM); 583 584 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 585 *mapp = map; 586 if (map == NULL) { 587 free(slist, M_BUSDMA); 588 return (ENOMEM); 589 } 590 591 /* 592 * If the tag's segments haven't been allocated yet we need to do it 593 * now, because we can't sleep for resources at map load time. 594 */ 595 if (dmat->segments == NULL) { 596 dmat->segments = malloc(dmat->nsegments * 597 sizeof(*dmat->segments), M_BUSDMA, M_NOWAIT); 598 if (dmat->segments == NULL) { 599 free(slist, M_BUSDMA); 600 uma_zfree(dmamap_zone, map); 601 *mapp = NULL; 602 return (ENOMEM); 603 } 604 } 605 606 /* 607 * Bouncing might be required if the driver asks for an active 608 * exclusion region, a data alignment that is stricter than 1, and/or 609 * an active address boundary. 610 */ 611 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 612 613 /* Must bounce */ 614 struct bounce_zone *bz; 615 int maxpages; 616 617 if (dmat->bounce_zone == NULL) { 618 if ((error = alloc_bounce_zone(dmat)) != 0) { 619 free(slist, M_BUSDMA); 620 uma_zfree(dmamap_zone, map); 621 *mapp = NULL; 622 return (error); 623 } 624 } 625 bz = dmat->bounce_zone; 626 627 /* Initialize the new map */ 628 STAILQ_INIT(&((*mapp)->bpages)); 629 630 /* 631 * Attempt to add pages to our pool on a per-instance 632 * basis up to a sane limit. 633 */ 634 maxpages = MAX_BPAGES; 635 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 636 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 637 int pages; 638 639 pages = MAX(atop(dmat->maxsize), 1); 640 pages = MIN(maxpages - bz->total_bpages, pages); 641 pages = MAX(pages, 1); 642 if (alloc_bounce_pages(dmat, pages) < pages) 643 error = ENOMEM; 644 645 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 646 if (error == 0) 647 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 648 } else { 649 error = 0; 650 } 651 } 652 bz->map_count++; 653 } 654 map->sync_count = 0; 655 map->slist = slist; 656 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 657 __func__, dmat, dmat->flags, error); 658 659 return (0); 660} 661 662/* 663 * Destroy a handle for mapping from kva/uva/physical 664 * address space into bus device space. 665 */ 666int 667bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 668{ 669 670 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 671 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 672 __func__, dmat, EBUSY); 673 return (EBUSY); 674 } 675 free(map->slist, M_BUSDMA); 676 uma_zfree(dmamap_zone, map); 677 if (dmat->bounce_zone) 678 dmat->bounce_zone->map_count--; 679 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 680 return (0); 681} 682 683/* 684 * Allocate a piece of memory that can be efficiently mapped into bus device 685 * space based on the constraints listed in the dma tag. Returns a pointer to 686 * the allocated memory, and a pointer to an associated bus_dmamap. 687 */ 688int 689bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 690 bus_dmamap_t *mapp) 691{ 692 struct sync_list *slist; 693 void * vaddr; 694 struct busdma_bufzone *bufzone; 695 busdma_bufalloc_t ba; 696 bus_dmamap_t map; 697 int mflags; 698 vm_memattr_t memattr; 699 700 if (flags & BUS_DMA_NOWAIT) 701 mflags = M_NOWAIT; 702 else 703 mflags = M_WAITOK; 704 /* 705 * If the tag's segments haven't been allocated yet we need to do it 706 * now, because we can't sleep for resources at map load time. 707 */ 708 if (dmat->segments == NULL) 709 dmat->segments = malloc(dmat->nsegments * 710 sizeof(*dmat->segments), M_BUSDMA, mflags); 711 712 slist = malloc(sizeof(*slist) * dmat->nsegments, M_BUSDMA, M_NOWAIT); 713 if (slist == NULL) 714 return (ENOMEM); 715 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 716 if (map == NULL) { 717 free(slist, M_BUSDMA); 718 return (ENOMEM); 719 } 720 if (flags & BUS_DMA_COHERENT) { 721 memattr = VM_MEMATTR_UNCACHEABLE; 722 ba = coherent_allocator; 723 map->flags |= DMAMAP_COHERENT; 724 } else { 725 memattr = VM_MEMATTR_DEFAULT; 726 ba = standard_allocator; 727 } 728 /* All buffers we allocate are cache-aligned. */ 729 map->flags |= DMAMAP_CACHE_ALIGNED; 730 731 if (flags & BUS_DMA_ZERO) 732 mflags |= M_ZERO; 733 734 /* 735 * Try to find a bufzone in the allocator that holds a cache of buffers 736 * of the right size for this request. If the buffer is too big to be 737 * held in the allocator cache, this returns NULL. 738 */ 739 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 740 741 /* 742 * Allocate the buffer from the uma(9) allocator if... 743 * - It's small enough to be in the allocator (bufzone not NULL). 744 * - The alignment constraint isn't larger than the allocation size 745 * (the allocator aligns buffers to their size boundaries). 746 * - There's no need to handle lowaddr/highaddr exclusion zones. 747 * else allocate non-contiguous pages if... 748 * - The page count that could get allocated doesn't exceed nsegments. 749 * - The alignment constraint isn't larger than a page boundary. 750 * - There are no boundary-crossing constraints. 751 * else allocate a block of contiguous pages because one or more of the 752 * constraints is something that only the contig allocator can fulfill. 753 */ 754 if (bufzone != NULL && dmat->alignment <= bufzone->size && 755 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 756 vaddr = uma_zalloc(bufzone->umazone, mflags); 757 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 758 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 759 vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 760 mflags, 0, dmat->lowaddr, memattr); 761 } else { 762 vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 763 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 764 memattr); 765 } 766 if (vaddr == NULL) { 767 free(slist, M_BUSDMA); 768 uma_zfree(dmamap_zone, map); 769 map = NULL; 770 } else { 771 map->slist = slist; 772 map->sync_count = 0; 773 } 774 *vaddrp = vaddr; 775 *mapp = map; 776 777 return (vaddr == NULL ? ENOMEM : 0); 778} 779 780/* 781 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 782 * its associated map. 783 */ 784void 785bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 786{ 787 struct busdma_bufzone *bufzone; 788 busdma_bufalloc_t ba; 789 790 if (map->flags & DMAMAP_COHERENT) 791 ba = coherent_allocator; 792 else 793 ba = standard_allocator; 794 795 free(map->slist, M_BUSDMA); 796 uma_zfree(dmamap_zone, map); 797 798 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 799 800 if (bufzone != NULL && dmat->alignment <= bufzone->size && 801 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 802 uma_zfree(bufzone->umazone, vaddr); 803 else 804 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 805} 806 807static void 808_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 809 bus_size_t buflen, int flags) 810{ 811 bus_addr_t curaddr; 812 bus_size_t sgsize; 813 814 if (map->pagesneeded == 0) { 815 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 816 dmat->lowaddr, dmat->boundary, dmat->alignment); 817 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 818 map, map->pagesneeded); 819 /* 820 * Count the number of bounce pages 821 * needed in order to complete this transfer 822 */ 823 curaddr = buf; 824 while (buflen != 0) { 825 sgsize = MIN(buflen, dmat->maxsegsz); 826 if (run_filter(dmat, curaddr) != 0) { 827 sgsize = MIN(sgsize, 828 PAGE_SIZE - (curaddr & PAGE_MASK)); 829 map->pagesneeded++; 830 } 831 curaddr += sgsize; 832 buflen -= sgsize; 833 } 834 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 835 } 836} 837 838static void 839_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 840 void *buf, bus_size_t buflen, int flags) 841{ 842 vm_offset_t vaddr; 843 vm_offset_t vendaddr; 844 bus_addr_t paddr; 845 846 if (map->pagesneeded == 0) { 847 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 848 dmat->lowaddr, dmat->boundary, dmat->alignment); 849 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 850 map, map->pagesneeded); 851 /* 852 * Count the number of bounce pages 853 * needed in order to complete this transfer 854 */ 855 vaddr = trunc_page((vm_offset_t)buf); 856 vendaddr = (vm_offset_t)buf + buflen; 857 858 while (vaddr < vendaddr) { 859 if (__predict_true(pmap == kernel_pmap)) 860 paddr = pmap_kextract(vaddr); 861 else 862 paddr = pmap_extract(pmap, vaddr); 863 if (run_filter(dmat, paddr) != 0) 864 map->pagesneeded++; 865 vaddr += PAGE_SIZE; 866 } 867 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 868 } 869} 870 871static int 872_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 873{ 874 875 /* Reserve Necessary Bounce Pages */ 876 mtx_lock(&bounce_lock); 877 if (flags & BUS_DMA_NOWAIT) { 878 if (reserve_bounce_pages(dmat, map, 0) != 0) { 879 mtx_unlock(&bounce_lock); 880 return (ENOMEM); 881 } 882 } else { 883 if (reserve_bounce_pages(dmat, map, 1) != 0) { 884 /* Queue us for resources */ 885 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 886 mtx_unlock(&bounce_lock); 887 return (EINPROGRESS); 888 } 889 } 890 mtx_unlock(&bounce_lock); 891 892 return (0); 893} 894 895/* 896 * Add a single contiguous physical range to the segment list. 897 */ 898static int 899_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 900 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 901{ 902 bus_addr_t baddr, bmask; 903 int seg; 904 905 /* 906 * Make sure we don't cross any boundaries. 907 */ 908 bmask = ~(dmat->boundary - 1); 909 if (dmat->boundary > 0) { 910 baddr = (curaddr + dmat->boundary) & bmask; 911 if (sgsize > (baddr - curaddr)) 912 sgsize = (baddr - curaddr); 913 } 914 if (dmat->ranges) { 915 struct arm32_dma_range *dr; 916 917 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 918 curaddr); 919 if (dr == NULL) 920 return (0); 921 /* 922 * In a valid DMA range. Translate the physical 923 * memory address to an address in the DMA window. 924 */ 925 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 926 927 } 928 929 seg = *segp; 930 /* 931 * Insert chunk into a segment, coalescing with 932 * the previous segment if possible. 933 */ 934 if (seg >= 0 && 935 curaddr == segs[seg].ds_addr + segs[seg].ds_len && 936 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 937 (dmat->boundary == 0 || 938 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 939 segs[seg].ds_len += sgsize; 940 } else { 941 if (++seg >= dmat->nsegments) 942 return (0); 943 segs[seg].ds_addr = curaddr; 944 segs[seg].ds_len = sgsize; 945 } 946 *segp = seg; 947 return (sgsize); 948} 949 950/* 951 * Utility function to load a physical buffer. segp contains 952 * the starting segment on entrace, and the ending segment on exit. 953 */ 954int 955_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 956 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 957{ 958 struct sync_list *sl; 959 bus_size_t sgsize; 960 bus_addr_t curaddr; 961 bus_addr_t sl_end = 0; 962 int error; 963 964 if (segs == NULL) 965 segs = dmat->segments; 966 967 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 968 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 969 if (map->pagesneeded != 0) { 970 error = _bus_dmamap_reserve_pages(dmat, map, flags); 971 if (error) 972 return (error); 973 } 974 } 975 976 sl = map->slist + map->sync_count - 1; 977 978 while (buflen > 0) { 979 curaddr = buf; 980 sgsize = MIN(buflen, dmat->maxsegsz); 981 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 982 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 983 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 984 curaddr = add_bounce_page(dmat, map, 0, curaddr, 985 sgsize); 986 } else { 987 if (map->sync_count > 0) 988 sl_end = VM_PAGE_TO_PHYS(sl->pages) + 989 sl->dataoffs + sl->datacount; 990 991 if (map->sync_count == 0 || curaddr != sl_end) { 992 if (++map->sync_count > dmat->nsegments) 993 break; 994 sl++; 995 sl->vaddr = 0; 996 sl->datacount = sgsize; 997 sl->pages = PHYS_TO_VM_PAGE(curaddr); 998 sl->dataoffs = curaddr & PAGE_MASK; 999 } else 1000 sl->datacount += sgsize; 1001 } 1002 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1003 segp); 1004 if (sgsize == 0) 1005 break; 1006 buf += sgsize; 1007 buflen -= sgsize; 1008 } 1009 1010 /* 1011 * Did we fit? 1012 */ 1013 if (buflen != 0) { 1014 _bus_dmamap_unload(dmat, map); 1015 return (EFBIG); /* XXX better return value here? */ 1016 } 1017 return (0); 1018} 1019 1020int 1021_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1022 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1023 bus_dma_segment_t *segs, int *segp) 1024{ 1025 1026 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1027 segs, segp)); 1028} 1029 1030/* 1031 * Utility function to load a linear buffer. segp contains 1032 * the starting segment on entrance, and the ending segment on exit. 1033 */ 1034int 1035_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1036 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1037 int *segp) 1038{ 1039 bus_size_t sgsize; 1040 bus_addr_t curaddr; 1041 bus_addr_t sl_pend = 0; 1042 struct sync_list *sl; 1043 vm_offset_t kvaddr; 1044 vm_offset_t vaddr = (vm_offset_t)buf; 1045 vm_offset_t sl_vend = 0; 1046 int error = 0; 1047 1048 if (segs == NULL) 1049 segs = dmat->segments; 1050 if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1051 map->flags |= DMAMAP_CACHE_ALIGNED; 1052 1053 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1054 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1055 if (map->pagesneeded != 0) { 1056 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1057 if (error) 1058 return (error); 1059 } 1060 } 1061 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1062 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1063 1064 sl = map->slist + map->sync_count - 1; 1065 1066 while (buflen > 0) { 1067 /* 1068 * Get the physical address for this segment. 1069 */ 1070 if (__predict_true(pmap == kernel_pmap)) { 1071 curaddr = pmap_kextract(vaddr); 1072 kvaddr = vaddr; 1073 } else { 1074 curaddr = pmap_extract(pmap, vaddr); 1075 map->flags &= ~DMAMAP_COHERENT; 1076 kvaddr = 0; 1077 } 1078 1079 /* 1080 * Compute the segment size, and adjust counts. 1081 */ 1082 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1083 if (sgsize > dmat->maxsegsz) 1084 sgsize = dmat->maxsegsz; 1085 if (buflen < sgsize) 1086 sgsize = buflen; 1087 1088 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1089 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1090 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1091 sgsize); 1092 } else { 1093 if (map->sync_count > 0) { 1094 sl_pend = VM_PAGE_TO_PHYS(sl->pages) + 1095 sl->dataoffs + sl->datacount; 1096 sl_vend = sl->vaddr + sl->datacount; 1097 } 1098 1099 if (map->sync_count == 0 || 1100 (kvaddr != 0 && kvaddr != sl_vend) || 1101 (kvaddr == 0 && curaddr != sl_pend)) { 1102 1103 if (++map->sync_count > dmat->nsegments) 1104 goto cleanup; 1105 sl++; 1106 sl->vaddr = kvaddr; 1107 sl->datacount = sgsize; 1108 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1109 sl->dataoffs = curaddr & PAGE_MASK; 1110 } else 1111 sl->datacount += sgsize; 1112 } 1113 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1114 segp); 1115 if (sgsize == 0) 1116 break; 1117 vaddr += sgsize; 1118 buflen -= sgsize; 1119 } 1120 1121cleanup: 1122 /* 1123 * Did we fit? 1124 */ 1125 if (buflen != 0) { 1126 _bus_dmamap_unload(dmat, map); 1127 return (EFBIG); /* XXX better return value here? */ 1128 } 1129 return (0); 1130} 1131 1132void 1133__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, 1134 bus_dmamap_callback_t *callback, void *callback_arg) 1135{ 1136 1137 KASSERT(dmat != NULL, ("dmatag is NULL")); 1138 KASSERT(map != NULL, ("dmamap is NULL")); 1139 map->mem = *mem; 1140 map->callback = callback; 1141 map->callback_arg = callback_arg; 1142} 1143 1144bus_dma_segment_t * 1145_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1146 bus_dma_segment_t *segs, int nsegs, int error) 1147{ 1148 1149 if (segs == NULL) 1150 segs = dmat->segments; 1151 return (segs); 1152} 1153 1154/* 1155 * Release the mapping held by map. 1156 */ 1157void 1158_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1159{ 1160 struct bounce_page *bpage; 1161 1162 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1163 STAILQ_REMOVE_HEAD(&map->bpages, links); 1164 free_bounce_page(dmat, bpage); 1165 } 1166 map->sync_count = 0; 1167} 1168 1169static void 1170bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1171 int bufaligned) 1172{ 1173 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1174 register_t s; 1175 int partial; 1176 1177 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1178 cpu_dcache_wb_range(buf, len); 1179 cpu_l2cache_wb_range(buf, len); 1180 } 1181 1182 /* 1183 * If the caller promises the buffer is properly aligned to a cache line 1184 * (even if the call parms make it look like it isn't) we can avoid 1185 * attempting to preserve the non-DMA part of the cache line in the 1186 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1187 * 1188 * This covers the case of mbufs, where we know how they're aligned and 1189 * know the CPU doesn't touch the header in front of the DMA data area 1190 * during the IO, but it may have touched it right before invoking the 1191 * sync, so a PREREAD writeback is required. 1192 * 1193 * It also handles buffers we created in bus_dmamem_alloc(), which are 1194 * always aligned and padded to cache line size even if the IO length 1195 * isn't a multiple of cache line size. In this case the PREREAD 1196 * writeback probably isn't required, but it's harmless. 1197 */ 1198 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1199 1200 if (op & BUS_DMASYNC_PREREAD) { 1201 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1202 cpu_dcache_inv_range(buf, len); 1203 cpu_l2cache_inv_range(buf, len); 1204 } else { 1205 cpu_dcache_wbinv_range(buf, len); 1206 cpu_l2cache_wbinv_range(buf, len); 1207 } 1208 } 1209 if (op & BUS_DMASYNC_POSTREAD) { 1210 if (partial && !bufaligned) { 1211 s = intr_disable(); 1212 if (buf & arm_dcache_align_mask) 1213 memcpy(_tmp_cl, (void *)(buf & 1214 ~arm_dcache_align_mask), 1215 buf & arm_dcache_align_mask); 1216 if ((buf + len) & arm_dcache_align_mask) 1217 memcpy(_tmp_clend, 1218 (void *)(buf + len), 1219 arm_dcache_align - 1220 ((buf + len) & arm_dcache_align_mask)); 1221 } 1222 cpu_dcache_inv_range(buf, len); 1223 cpu_l2cache_inv_range(buf, len); 1224 if (partial && !bufaligned) { 1225 if (buf & arm_dcache_align_mask) 1226 memcpy((void *)(buf & 1227 ~arm_dcache_align_mask), _tmp_cl, 1228 buf & arm_dcache_align_mask); 1229 if ((buf + len) & arm_dcache_align_mask) 1230 memcpy((void *)(buf + len), 1231 _tmp_clend, arm_dcache_align - 1232 ((buf + len) & arm_dcache_align_mask)); 1233 intr_restore(s); 1234 } 1235 } 1236} 1237 1238static void 1239bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 1240 int bufaligned) 1241{ 1242 vm_offset_t tempvaddr; 1243 vm_page_t curpage; 1244 size_t npages; 1245 1246 if (sl->vaddr != 0) { 1247 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); 1248 return; 1249 } 1250 1251 tempvaddr = 0; 1252 npages = atop(round_page(sl->dataoffs + sl->datacount)); 1253 1254 for (curpage = sl->pages; curpage != sl->pages + npages; ++curpage) { 1255 /* 1256 * If the page is mapped to some other VA that hasn't 1257 * been supplied to busdma, then pmap_quick_enter_page() 1258 * will find all duplicate mappings and mark them 1259 * uncacheable. 1260 * That will also do any necessary wb/inv. Otherwise, 1261 * if the page is truly unmapped, then we don't actually 1262 * need to do cache maintenance. 1263 * XXX: May overwrite DMA'ed data in the POSTREAD 1264 * case where the CPU has written to a cacheline not 1265 * completely covered by the DMA region. 1266 */ 1267 KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) + 1268 ptoa(curpage - sl->pages), 1269 ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1270 VM_PAGE_TO_PHYS(curpage), VM_PAGE_TO_PHYS(sl->pages) + 1271 ptoa(curpage - sl->pages))); 1272 tempvaddr = pmap_quick_enter_page(curpage); 1273 pmap_quick_remove_page(tempvaddr); 1274 } 1275} 1276 1277static void 1278_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1279{ 1280 struct bounce_page *bpage; 1281 vm_offset_t datavaddr, tempvaddr; 1282 1283 if ((op & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) == 0) 1284 return; 1285 1286 STAILQ_FOREACH(bpage, &map->bpages, links) { 1287 tempvaddr = 0; 1288 datavaddr = bpage->datavaddr; 1289 if (op & BUS_DMASYNC_PREWRITE) { 1290 if (datavaddr == 0) { 1291 tempvaddr = 1292 pmap_quick_enter_page(bpage->datapage); 1293 datavaddr = tempvaddr | bpage->dataoffs; 1294 } 1295 bcopy((void *)datavaddr, 1296 (void *)bpage->vaddr, bpage->datacount); 1297 if (tempvaddr != 0) 1298 pmap_quick_remove_page(tempvaddr); 1299 cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1300 cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1301 dmat->bounce_zone->total_bounced++; 1302 } 1303 if (op & BUS_DMASYNC_POSTREAD) { 1304 cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1305 cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1306 if (datavaddr == 0) { 1307 tempvaddr = 1308 pmap_quick_enter_page(bpage->datapage); 1309 datavaddr = tempvaddr | bpage->dataoffs; 1310 } 1311 bcopy((void *)bpage->vaddr, 1312 (void *)datavaddr, bpage->datacount); 1313 if (tempvaddr != 0) 1314 pmap_quick_remove_page(tempvaddr); 1315 dmat->bounce_zone->total_bounced++; 1316 } 1317 } 1318} 1319 1320void 1321_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1322{ 1323 struct sync_list *sl, *end; 1324 int bufaligned; 1325 1326 if (op == BUS_DMASYNC_POSTWRITE) 1327 return; 1328 if (map->flags & DMAMAP_COHERENT) 1329 goto drain; 1330 if (STAILQ_FIRST(&map->bpages)) 1331 _bus_dmamap_sync_bp(dmat, map, op); 1332 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1333 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1334 if (map->sync_count) { 1335 end = &map->slist[map->sync_count]; 1336 for (sl = &map->slist[0]; sl != end; sl++) 1337 bus_dmamap_sync_sl(sl, op, bufaligned); 1338 } 1339 1340drain: 1341 1342 cpu_drain_writebuf(); 1343} 1344 1345static void 1346init_bounce_pages(void *dummy __unused) 1347{ 1348 1349 total_bpages = 0; 1350 STAILQ_INIT(&bounce_zone_list); 1351 STAILQ_INIT(&bounce_map_waitinglist); 1352 STAILQ_INIT(&bounce_map_callbacklist); 1353 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1354} 1355SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1356 1357static struct sysctl_ctx_list * 1358busdma_sysctl_tree(struct bounce_zone *bz) 1359{ 1360 1361 return (&bz->sysctl_tree); 1362} 1363 1364static struct sysctl_oid * 1365busdma_sysctl_tree_top(struct bounce_zone *bz) 1366{ 1367 1368 return (bz->sysctl_tree_top); 1369} 1370 1371static int 1372alloc_bounce_zone(bus_dma_tag_t dmat) 1373{ 1374 struct bounce_zone *bz; 1375 1376 /* Check to see if we already have a suitable zone */ 1377 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1378 if ((dmat->alignment <= bz->alignment) && 1379 (dmat->lowaddr >= bz->lowaddr)) { 1380 dmat->bounce_zone = bz; 1381 return (0); 1382 } 1383 } 1384 1385 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA, 1386 M_NOWAIT | M_ZERO)) == NULL) 1387 return (ENOMEM); 1388 1389 STAILQ_INIT(&bz->bounce_page_list); 1390 bz->free_bpages = 0; 1391 bz->reserved_bpages = 0; 1392 bz->active_bpages = 0; 1393 bz->lowaddr = dmat->lowaddr; 1394 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1395 bz->map_count = 0; 1396 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1397 busdma_zonecount++; 1398 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1399 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1400 dmat->bounce_zone = bz; 1401 1402 sysctl_ctx_init(&bz->sysctl_tree); 1403 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1404 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1405 CTLFLAG_RD, 0, ""); 1406 if (bz->sysctl_tree_top == NULL) { 1407 sysctl_ctx_free(&bz->sysctl_tree); 1408 return (0); /* XXX error code? */ 1409 } 1410 1411 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1412 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1413 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1414 "Total bounce pages"); 1415 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1416 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1417 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1418 "Free bounce pages"); 1419 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1420 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1421 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1422 "Reserved bounce pages"); 1423 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1424 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1425 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1426 "Active bounce pages"); 1427 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1428 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1429 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1430 "Total bounce requests (pages bounced)"); 1431 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1432 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1433 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1434 "Total bounce requests that were deferred"); 1435 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1436 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1437 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1438 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1439 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1440 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1441 1442 return (0); 1443} 1444 1445static int 1446alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1447{ 1448 struct bounce_zone *bz; 1449 int count; 1450 1451 bz = dmat->bounce_zone; 1452 count = 0; 1453 while (numpages > 0) { 1454 struct bounce_page *bpage; 1455 1456 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_BUSDMA, 1457 M_NOWAIT | M_ZERO); 1458 1459 if (bpage == NULL) 1460 break; 1461 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE, 1462 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1463 if (bpage->vaddr == 0) { 1464 free(bpage, M_BUSDMA); 1465 break; 1466 } 1467 bpage->busaddr = pmap_kextract(bpage->vaddr); 1468 mtx_lock(&bounce_lock); 1469 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1470 total_bpages++; 1471 bz->total_bpages++; 1472 bz->free_bpages++; 1473 mtx_unlock(&bounce_lock); 1474 count++; 1475 numpages--; 1476 } 1477 return (count); 1478} 1479 1480static int 1481reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1482{ 1483 struct bounce_zone *bz; 1484 int pages; 1485 1486 mtx_assert(&bounce_lock, MA_OWNED); 1487 bz = dmat->bounce_zone; 1488 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1489 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1490 return (map->pagesneeded - (map->pagesreserved + pages)); 1491 bz->free_bpages -= pages; 1492 bz->reserved_bpages += pages; 1493 map->pagesreserved += pages; 1494 pages = map->pagesneeded - map->pagesreserved; 1495 1496 return (pages); 1497} 1498 1499static bus_addr_t 1500add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1501 bus_addr_t addr, bus_size_t size) 1502{ 1503 struct bounce_zone *bz; 1504 struct bounce_page *bpage; 1505 1506 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1507 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1508 1509 bz = dmat->bounce_zone; 1510 if (map->pagesneeded == 0) 1511 panic("add_bounce_page: map doesn't need any pages"); 1512 map->pagesneeded--; 1513 1514 if (map->pagesreserved == 0) 1515 panic("add_bounce_page: map doesn't need any pages"); 1516 map->pagesreserved--; 1517 1518 mtx_lock(&bounce_lock); 1519 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1520 if (bpage == NULL) 1521 panic("add_bounce_page: free page list is empty"); 1522 1523 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1524 bz->reserved_bpages--; 1525 bz->active_bpages++; 1526 mtx_unlock(&bounce_lock); 1527 1528 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1529 /* Page offset needs to be preserved. */ 1530 bpage->vaddr |= addr & PAGE_MASK; 1531 bpage->busaddr |= addr & PAGE_MASK; 1532 } 1533 bpage->datavaddr = vaddr; 1534 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1535 bpage->dataoffs = addr & PAGE_MASK; 1536 bpage->datacount = size; 1537 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1538 return (bpage->busaddr); 1539} 1540 1541static void 1542free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1543{ 1544 struct bus_dmamap *map; 1545 struct bounce_zone *bz; 1546 1547 bz = dmat->bounce_zone; 1548 bpage->datavaddr = 0; 1549 bpage->datacount = 0; 1550 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1551 /* 1552 * Reset the bounce page to start at offset 0. Other uses 1553 * of this bounce page may need to store a full page of 1554 * data and/or assume it starts on a page boundary. 1555 */ 1556 bpage->vaddr &= ~PAGE_MASK; 1557 bpage->busaddr &= ~PAGE_MASK; 1558 } 1559 1560 mtx_lock(&bounce_lock); 1561 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1562 bz->free_bpages++; 1563 bz->active_bpages--; 1564 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1565 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1566 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1567 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1568 map, links); 1569 busdma_swi_pending = 1; 1570 bz->total_deferred++; 1571 swi_sched(vm_ih, 0); 1572 } 1573 } 1574 mtx_unlock(&bounce_lock); 1575} 1576 1577void 1578busdma_swi(void) 1579{ 1580 bus_dma_tag_t dmat; 1581 struct bus_dmamap *map; 1582 1583 mtx_lock(&bounce_lock); 1584 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1585 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1586 mtx_unlock(&bounce_lock); 1587 dmat = map->dmat; 1588 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1589 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1590 map->callback_arg, BUS_DMA_WAITOK); 1591 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1592 mtx_lock(&bounce_lock); 1593 } 1594 mtx_unlock(&bounce_lock); 1595} 1596