busdma_machdep-v6.c revision 273736
1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2010 Mark Tinguely 4 * Copyright (c) 2004 Olivier Houchard 5 * Copyright (c) 2002 Peter Grehan 6 * Copyright (c) 1997, 1998 Justin T. Gibbs. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions, and the following disclaimer, 14 * without modification, immediately at the beginning of the file. 15 * 2. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 22 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD: stable/10/sys/arm/arm/busdma_machdep-v6.c 273736 2014-10-27 14:38:00Z hselasky $"); 35 36#define _ARM32_BUS_DMA_PRIVATE 37#include <sys/param.h> 38#include <sys/kdb.h> 39#include <ddb/ddb.h> 40#include <ddb/db_output.h> 41#include <sys/systm.h> 42#include <sys/malloc.h> 43#include <sys/bus.h> 44#include <sys/busdma_bufalloc.h> 45#include <sys/interrupt.h> 46#include <sys/kernel.h> 47#include <sys/ktr.h> 48#include <sys/lock.h> 49#include <sys/memdesc.h> 50#include <sys/proc.h> 51#include <sys/mutex.h> 52#include <sys/sysctl.h> 53#include <sys/uio.h> 54 55#include <vm/vm.h> 56#include <vm/vm_page.h> 57#include <vm/vm_map.h> 58#include <vm/vm_extern.h> 59#include <vm/vm_kern.h> 60 61#include <machine/atomic.h> 62#include <machine/bus.h> 63#include <machine/cpufunc.h> 64#include <machine/md_var.h> 65 66#define MAX_BPAGES 64 67#define MAX_DMA_SEGMENTS 4096 68#define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2 69#define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3 70#define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE) 71#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 72 73struct bounce_zone; 74 75struct bus_dma_tag { 76 bus_dma_tag_t parent; 77 bus_size_t alignment; 78 bus_size_t boundary; 79 bus_addr_t lowaddr; 80 bus_addr_t highaddr; 81 bus_dma_filter_t *filter; 82 void *filterarg; 83 bus_size_t maxsize; 84 u_int nsegments; 85 bus_size_t maxsegsz; 86 int flags; 87 int ref_count; 88 int map_count; 89 bus_dma_lock_t *lockfunc; 90 void *lockfuncarg; 91 struct bounce_zone *bounce_zone; 92 /* 93 * DMA range for this tag. If the page doesn't fall within 94 * one of these ranges, an error is returned. The caller 95 * may then decide what to do with the transfer. If the 96 * range pointer is NULL, it is ignored. 97 */ 98 struct arm32_dma_range *ranges; 99 int _nranges; 100}; 101 102struct bounce_page { 103 vm_offset_t vaddr; /* kva of bounce buffer */ 104 bus_addr_t busaddr; /* Physical address */ 105 vm_offset_t datavaddr; /* kva of client data */ 106 bus_addr_t dataaddr; /* client physical address */ 107 bus_size_t datacount; /* client data count */ 108 STAILQ_ENTRY(bounce_page) links; 109}; 110 111struct sync_list { 112 vm_offset_t vaddr; /* kva of bounce buffer */ 113 bus_addr_t busaddr; /* Physical address */ 114 bus_size_t datacount; /* client data count */ 115}; 116 117int busdma_swi_pending; 118 119struct bounce_zone { 120 STAILQ_ENTRY(bounce_zone) links; 121 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 122 int total_bpages; 123 int free_bpages; 124 int reserved_bpages; 125 int active_bpages; 126 int total_bounced; 127 int total_deferred; 128 int map_count; 129 bus_size_t alignment; 130 bus_addr_t lowaddr; 131 char zoneid[8]; 132 char lowaddrid[20]; 133 struct sysctl_ctx_list sysctl_tree; 134 struct sysctl_oid *sysctl_tree_top; 135}; 136 137static struct mtx bounce_lock; 138static int total_bpages; 139static int busdma_zonecount; 140static uint32_t tags_total; 141static uint32_t maps_total; 142static uint32_t maps_dmamem; 143static uint32_t maps_coherent; 144static uint64_t maploads_total; 145static uint64_t maploads_bounced; 146static uint64_t maploads_coherent; 147static uint64_t maploads_dmamem; 148static uint64_t maploads_mbuf; 149static uint64_t maploads_physmem; 150 151static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 152 153SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 154SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0, 155 "Number of active tags"); 156SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0, 157 "Number of active maps"); 158SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0, 159 "Number of active maps for bus_dmamem_alloc buffers"); 160SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0, 161 "Number of active maps with BUS_DMA_COHERENT flag set"); 162SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, &maploads_total, 0, 163 "Number of load operations performed"); 164SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, &maploads_bounced, 0, 165 "Number of load operations that used bounce buffers"); 166SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, &maploads_dmamem, 0, 167 "Number of load operations on BUS_DMA_COHERENT memory"); 168SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, &maploads_dmamem, 0, 169 "Number of load operations on bus_dmamem_alloc buffers"); 170SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, &maploads_mbuf, 0, 171 "Number of load operations for mbufs"); 172SYSCTL_UQUAD(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, &maploads_physmem, 0, 173 "Number of load operations on physical buffers"); 174SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 175 "Total bounce pages"); 176 177struct bus_dmamap { 178 struct bp_list bpages; 179 int pagesneeded; 180 int pagesreserved; 181 bus_dma_tag_t dmat; 182 struct memdesc mem; 183 pmap_t pmap; 184 bus_dmamap_callback_t *callback; 185 void *callback_arg; 186 int flags; 187#define DMAMAP_COHERENT (1 << 0) 188#define DMAMAP_DMAMEM_ALLOC (1 << 1) 189#define DMAMAP_MBUF (1 << 2) 190 STAILQ_ENTRY(bus_dmamap) links; 191 bus_dma_segment_t *segments; 192 int sync_count; 193 struct sync_list slist[]; 194}; 195 196static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 197static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 198 199static void init_bounce_pages(void *dummy); 200static int alloc_bounce_zone(bus_dma_tag_t dmat); 201static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 202static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 203 int commit); 204static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 205 vm_offset_t vaddr, bus_addr_t addr, 206 bus_size_t size); 207static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 208static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 209 void *buf, bus_size_t buflen, int flags); 210static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 211 vm_paddr_t buf, bus_size_t buflen, int flags); 212static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 213 int flags); 214 215static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 216static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 217static void 218busdma_init(void *dummy) 219{ 220 int uma_flags; 221 222 uma_flags = 0; 223 224 /* Create a cache of buffers in standard (cacheable) memory. */ 225 standard_allocator = busdma_bufalloc_create("buffer", 226 arm_dcache_align, /* minimum_alignment */ 227 NULL, /* uma_alloc func */ 228 NULL, /* uma_free func */ 229 uma_flags); /* uma_zcreate_flags */ 230 231#ifdef INVARIANTS 232 /* 233 * Force UMA zone to allocate service structures like 234 * slabs using own allocator. uma_debug code performs 235 * atomic ops on uma_slab_t fields and safety of this 236 * operation is not guaranteed for write-back caches 237 */ 238 uma_flags = UMA_ZONE_OFFPAGE; 239#endif 240 /* 241 * Create a cache of buffers in uncacheable memory, to implement the 242 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 243 */ 244 coherent_allocator = busdma_bufalloc_create("coherent", 245 arm_dcache_align, /* minimum_alignment */ 246 busdma_bufalloc_alloc_uncacheable, 247 busdma_bufalloc_free_uncacheable, 248 uma_flags); /* uma_zcreate_flags */ 249} 250 251/* 252 * This init historically used SI_SUB_VM, but now the init code requires 253 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 254 * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 255 * SI_SUB_KMEM and SI_ORDER_THIRD. 256 */ 257SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 258 259static int 260exclusion_bounce_check(vm_offset_t lowaddr, vm_offset_t highaddr) 261{ 262 int i; 263 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 264 if ((lowaddr >= phys_avail[i] && lowaddr < phys_avail[i + 1]) || 265 (lowaddr < phys_avail[i] && highaddr >= phys_avail[i])) 266 return (1); 267 } 268 return (0); 269} 270 271/* 272 * Return true if the tag has an exclusion zone that could lead to bouncing. 273 */ 274static __inline int 275exclusion_bounce(bus_dma_tag_t dmat) 276{ 277 278 return (dmat->flags & BUS_DMA_EXCL_BOUNCE); 279} 280 281/* 282 * Return true if the given address does not fall on the alignment boundary. 283 */ 284static __inline int 285alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 286{ 287 288 return (addr & (dmat->alignment - 1)); 289} 290 291/* 292 * Return true if the DMA should bounce because the start or end does not fall 293 * on a cacheline boundary (which would require a partial cacheline flush). 294 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 295 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 296 * strict rule that such memory cannot be accessed by the CPU while DMA is in 297 * progress (or by multiple DMA engines at once), so that it's always safe to do 298 * full cacheline flushes even if that affects memory outside the range of a 299 * given DMA operation that doesn't involve the full allocated buffer. If we're 300 * mapping an mbuf, that follows the same rules as a buffer we allocated. 301 */ 302static __inline int 303cacheline_bounce(bus_dmamap_t map, bus_addr_t addr, bus_size_t size) 304{ 305 306 if (map->flags & (DMAMAP_DMAMEM_ALLOC | DMAMAP_COHERENT | DMAMAP_MBUF)) 307 return (0); 308 return ((addr | size) & arm_dcache_align_mask); 309} 310 311/* 312 * Return true if we might need to bounce the DMA described by addr and size. 313 * 314 * This is used to quick-check whether we need to do the more expensive work of 315 * checking the DMA page-by-page looking for alignment and exclusion bounces. 316 * 317 * Note that the addr argument might be either virtual or physical. It doesn't 318 * matter because we only look at the low-order bits, which are the same in both 319 * address spaces. 320 */ 321static __inline int 322might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t addr, 323 bus_size_t size) 324{ 325 return ((dmat->flags & BUS_DMA_EXCL_BOUNCE) || 326 alignment_bounce(dmat, addr) || 327 cacheline_bounce(map, addr, size)); 328} 329 330/* 331 * Return true if we must bounce the DMA described by paddr and size. 332 * 333 * Bouncing can be triggered by DMA that doesn't begin and end on cacheline 334 * boundaries, or doesn't begin on an alignment boundary, or falls within the 335 * exclusion zone of any tag in the ancestry chain. 336 * 337 * For exclusions, walk the chain of tags comparing paddr to the exclusion zone 338 * within each tag. If the tag has a filter function, use it to decide whether 339 * the DMA needs to bounce, otherwise any DMA within the zone bounces. 340 */ 341static int 342must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 343 bus_size_t size) 344{ 345 346 if (cacheline_bounce(map, paddr, size)) 347 return (1); 348 349 /* 350 * The tag already contains ancestors' alignment restrictions so this 351 * check doesn't need to be inside the loop. 352 */ 353 if (alignment_bounce(dmat, paddr)) 354 return (1); 355 356 /* 357 * Even though each tag has an exclusion zone that is a superset of its 358 * own and all its ancestors' exclusions, the exclusion zone of each tag 359 * up the chain must be checked within the loop, because the busdma 360 * rules say the filter function is called only when the address lies 361 * within the low-highaddr range of the tag that filterfunc belongs to. 362 */ 363 while (dmat != NULL && exclusion_bounce(dmat)) { 364 if ((paddr >= dmat->lowaddr && paddr <= dmat->highaddr) && 365 (dmat->filter == NULL || 366 dmat->filter(dmat->filterarg, paddr) != 0)) 367 return (1); 368 dmat = dmat->parent; 369 } 370 371 return (0); 372} 373 374static __inline struct arm32_dma_range * 375_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 376 bus_addr_t curaddr) 377{ 378 struct arm32_dma_range *dr; 379 int i; 380 381 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 382 if (curaddr >= dr->dr_sysbase && 383 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 384 return (dr); 385 } 386 387 return (NULL); 388} 389 390/* 391 * Convenience function for manipulating driver locks from busdma (during 392 * busdma_swi, for example). Drivers that don't provide their own locks 393 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 394 * non-mutex locking scheme don't have to use this at all. 395 */ 396void 397busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 398{ 399 struct mtx *dmtx; 400 401 dmtx = (struct mtx *)arg; 402 switch (op) { 403 case BUS_DMA_LOCK: 404 mtx_lock(dmtx); 405 break; 406 case BUS_DMA_UNLOCK: 407 mtx_unlock(dmtx); 408 break; 409 default: 410 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 411 } 412} 413 414/* 415 * dflt_lock should never get called. It gets put into the dma tag when 416 * lockfunc == NULL, which is only valid if the maps that are associated 417 * with the tag are meant to never be defered. 418 * XXX Should have a way to identify which driver is responsible here. 419 */ 420static void 421dflt_lock(void *arg, bus_dma_lock_op_t op) 422{ 423 panic("driver error: busdma dflt_lock called"); 424} 425 426/* 427 * Allocate a device specific dma_tag. 428 */ 429int 430bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 431 bus_size_t boundary, bus_addr_t lowaddr, 432 bus_addr_t highaddr, bus_dma_filter_t *filter, 433 void *filterarg, bus_size_t maxsize, int nsegments, 434 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 435 void *lockfuncarg, bus_dma_tag_t *dmat) 436{ 437 bus_dma_tag_t newtag; 438 int error = 0; 439 440#if 0 441 if (!parent) 442 parent = arm_root_dma_tag; 443#endif 444 445 /* Basic sanity checking */ 446 if (boundary != 0 && boundary < maxsegsz) 447 maxsegsz = boundary; 448 449 /* Return a NULL tag on failure */ 450 *dmat = NULL; 451 452 if (maxsegsz == 0) { 453 return (EINVAL); 454 } 455 456 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 457 M_ZERO | M_NOWAIT); 458 if (newtag == NULL) { 459 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 460 __func__, newtag, 0, error); 461 return (ENOMEM); 462 } 463 464 newtag->parent = parent; 465 newtag->alignment = alignment; 466 newtag->boundary = boundary; 467 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 468 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 469 (PAGE_SIZE - 1); 470 newtag->filter = filter; 471 newtag->filterarg = filterarg; 472 newtag->maxsize = maxsize; 473 newtag->nsegments = nsegments; 474 newtag->maxsegsz = maxsegsz; 475 newtag->flags = flags; 476 newtag->ref_count = 1; /* Count ourself */ 477 newtag->map_count = 0; 478 newtag->ranges = bus_dma_get_range(); 479 newtag->_nranges = bus_dma_get_range_nb(); 480 if (lockfunc != NULL) { 481 newtag->lockfunc = lockfunc; 482 newtag->lockfuncarg = lockfuncarg; 483 } else { 484 newtag->lockfunc = dflt_lock; 485 newtag->lockfuncarg = NULL; 486 } 487 488 /* Take into account any restrictions imposed by our parent tag */ 489 if (parent != NULL) { 490 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 491 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 492 newtag->alignment = MAX(parent->alignment, newtag->alignment); 493 newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; 494 if (newtag->boundary == 0) 495 newtag->boundary = parent->boundary; 496 else if (parent->boundary != 0) 497 newtag->boundary = MIN(parent->boundary, 498 newtag->boundary); 499 if (newtag->filter == NULL) { 500 /* 501 * Short circuit to looking at our parent directly 502 * since we have encapsulated all of its information 503 */ 504 newtag->filter = parent->filter; 505 newtag->filterarg = parent->filterarg; 506 newtag->parent = parent->parent; 507 } 508 if (newtag->parent != NULL) 509 atomic_add_int(&parent->ref_count, 1); 510 } 511 512 if (exclusion_bounce_check(newtag->lowaddr, newtag->highaddr)) 513 newtag->flags |= BUS_DMA_EXCL_BOUNCE; 514 if (alignment_bounce(newtag, 1)) 515 newtag->flags |= BUS_DMA_ALIGN_BOUNCE; 516 517 /* 518 * Any request can auto-bounce due to cacheline alignment, in addition 519 * to any alignment or boundary specifications in the tag, so if the 520 * ALLOCNOW flag is set, there's always work to do. 521 */ 522 if ((flags & BUS_DMA_ALLOCNOW) != 0) { 523 struct bounce_zone *bz; 524 /* 525 * Round size up to a full page, and add one more page because 526 * there can always be one more boundary crossing than the 527 * number of pages in a transfer. 528 */ 529 maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE; 530 531 if ((error = alloc_bounce_zone(newtag)) != 0) { 532 free(newtag, M_DEVBUF); 533 return (error); 534 } 535 bz = newtag->bounce_zone; 536 537 if (ptoa(bz->total_bpages) < maxsize) { 538 int pages; 539 540 pages = atop(maxsize) - bz->total_bpages; 541 542 /* Add pages to our bounce pool */ 543 if (alloc_bounce_pages(newtag, pages) < pages) 544 error = ENOMEM; 545 } 546 /* Performed initial allocation */ 547 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 548 } else 549 newtag->bounce_zone = NULL; 550 551 if (error != 0) { 552 free(newtag, M_DEVBUF); 553 } else { 554 atomic_add_32(&tags_total, 1); 555 *dmat = newtag; 556 } 557 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 558 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 559 return (error); 560} 561 562int 563bus_dma_tag_destroy(bus_dma_tag_t dmat) 564{ 565 bus_dma_tag_t dmat_copy; 566 int error; 567 568 error = 0; 569 dmat_copy = dmat; 570 571 if (dmat != NULL) { 572 573 if (dmat->map_count != 0) { 574 error = EBUSY; 575 goto out; 576 } 577 578 while (dmat != NULL) { 579 bus_dma_tag_t parent; 580 581 parent = dmat->parent; 582 atomic_subtract_int(&dmat->ref_count, 1); 583 if (dmat->ref_count == 0) { 584 atomic_subtract_32(&tags_total, 1); 585 free(dmat, M_DEVBUF); 586 /* 587 * Last reference count, so 588 * release our reference 589 * count on our parent. 590 */ 591 dmat = parent; 592 } else 593 dmat = NULL; 594 } 595 } 596out: 597 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 598 return (error); 599} 600 601static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp) 602{ 603 struct bounce_zone *bz; 604 int maxpages; 605 int error; 606 607 if (dmat->bounce_zone == NULL) 608 if ((error = alloc_bounce_zone(dmat)) != 0) 609 return (error); 610 bz = dmat->bounce_zone; 611 /* Initialize the new map */ 612 STAILQ_INIT(&(mapp->bpages)); 613 614 /* 615 * Attempt to add pages to our pool on a per-instance basis up to a sane 616 * limit. Even if the tag isn't flagged as COULD_BOUNCE due to 617 * alignment and boundary constraints, it could still auto-bounce due to 618 * cacheline alignment, which requires at most two bounce pages. 619 */ 620 if (dmat->flags & BUS_DMA_COULD_BOUNCE) 621 maxpages = MAX_BPAGES; 622 else 623 maxpages = 2 * bz->map_count; 624 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || 625 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 626 int pages; 627 628 pages = atop(roundup2(dmat->maxsize, PAGE_SIZE)) + 1; 629 pages = MIN(maxpages - bz->total_bpages, pages); 630 pages = MAX(pages, 2); 631 if (alloc_bounce_pages(dmat, pages) < pages) 632 return (ENOMEM); 633 634 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) 635 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 636 } 637 bz->map_count++; 638 return (0); 639} 640 641static bus_dmamap_t 642allocate_map(bus_dma_tag_t dmat, int mflags) 643{ 644 int mapsize, segsize; 645 bus_dmamap_t map; 646 647 /* 648 * Allocate the map. The map structure ends with an embedded 649 * variable-sized array of sync_list structures. Following that 650 * we allocate enough extra space to hold the array of bus_dma_segments. 651 */ 652 KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, 653 ("cannot allocate %u dma segments (max is %u)", 654 dmat->nsegments, MAX_DMA_SEGMENTS)); 655 segsize = sizeof(struct bus_dma_segment) * dmat->nsegments; 656 mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments; 657 map = malloc(mapsize + segsize, M_DEVBUF, mflags | M_ZERO); 658 if (map == NULL) { 659 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 660 return (NULL); 661 } 662 map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize); 663 return (map); 664} 665 666/* 667 * Allocate a handle for mapping from kva/uva/physical 668 * address space into bus device space. 669 */ 670int 671bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 672{ 673 bus_dmamap_t map; 674 int error = 0; 675 676 *mapp = map = allocate_map(dmat, M_NOWAIT); 677 if (map == NULL) { 678 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 679 return (ENOMEM); 680 } 681 682 /* 683 * Bouncing might be required if the driver asks for an exclusion 684 * region, a data alignment that is stricter than 1, or DMA that begins 685 * or ends with a partial cacheline. Whether bouncing will actually 686 * happen can't be known until mapping time, but we need to pre-allocate 687 * resources now because we might not be allowed to at mapping time. 688 */ 689 error = allocate_bz_and_pages(dmat, map); 690 if (error != 0) { 691 free(map, M_DEVBUF); 692 *mapp = NULL; 693 return (error); 694 } 695 if (map->flags & DMAMAP_COHERENT) 696 atomic_add_32(&maps_coherent, 1); 697 atomic_add_32(&maps_total, 1); 698 return (0); 699} 700 701/* 702 * Destroy a handle for mapping from kva/uva/physical 703 * address space into bus device space. 704 */ 705int 706bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 707{ 708 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 709 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 710 __func__, dmat, EBUSY); 711 return (EBUSY); 712 } 713 if (dmat->bounce_zone) 714 dmat->bounce_zone->map_count--; 715 if (map->flags & DMAMAP_COHERENT) 716 atomic_subtract_32(&maps_coherent, 1); 717 atomic_subtract_32(&maps_total, 1); 718 free(map, M_DEVBUF); 719 dmat->map_count--; 720 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 721 return (0); 722} 723 724 725/* 726 * Allocate a piece of memory that can be efficiently mapped into 727 * bus device space based on the constraints lited in the dma tag. 728 * A dmamap to for use with dmamap_load is also allocated. 729 */ 730int 731bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 732 bus_dmamap_t *mapp) 733{ 734 busdma_bufalloc_t ba; 735 struct busdma_bufzone *bufzone; 736 bus_dmamap_t map; 737 vm_memattr_t memattr; 738 int mflags; 739 740 if (flags & BUS_DMA_NOWAIT) 741 mflags = M_NOWAIT; 742 else 743 mflags = M_WAITOK; 744 if (flags & BUS_DMA_ZERO) 745 mflags |= M_ZERO; 746 747 *mapp = map = allocate_map(dmat, mflags); 748 if (map == NULL) { 749 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 750 __func__, dmat, dmat->flags, ENOMEM); 751 return (ENOMEM); 752 } 753 map->flags = DMAMAP_DMAMEM_ALLOC; 754 755 /* Choose a busdma buffer allocator based on memory type flags. */ 756 if (flags & BUS_DMA_COHERENT) { 757 memattr = VM_MEMATTR_UNCACHEABLE; 758 ba = coherent_allocator; 759 map->flags |= DMAMAP_COHERENT; 760 } else { 761 memattr = VM_MEMATTR_DEFAULT; 762 ba = standard_allocator; 763 } 764 765 /* 766 * Try to find a bufzone in the allocator that holds a cache of buffers 767 * of the right size for this request. If the buffer is too big to be 768 * held in the allocator cache, this returns NULL. 769 */ 770 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 771 772 /* 773 * Allocate the buffer from the uma(9) allocator if... 774 * - It's small enough to be in the allocator (bufzone not NULL). 775 * - The alignment constraint isn't larger than the allocation size 776 * (the allocator aligns buffers to their size boundaries). 777 * - There's no need to handle lowaddr/highaddr exclusion zones. 778 * else allocate non-contiguous pages if... 779 * - The page count that could get allocated doesn't exceed nsegments. 780 * - The alignment constraint isn't larger than a page boundary. 781 * - There are no boundary-crossing constraints. 782 * else allocate a block of contiguous pages because one or more of the 783 * constraints is something that only the contig allocator can fulfill. 784 */ 785 if (bufzone != NULL && dmat->alignment <= bufzone->size && 786 !exclusion_bounce(dmat)) { 787 *vaddr = uma_zalloc(bufzone->umazone, mflags); 788 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 789 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 790 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 791 mflags, 0, dmat->lowaddr, memattr); 792 } else { 793 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 794 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 795 memattr); 796 } 797 798 799 if (*vaddr == NULL) { 800 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 801 __func__, dmat, dmat->flags, ENOMEM); 802 free(map, M_DEVBUF); 803 *mapp = NULL; 804 return (ENOMEM); 805 } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { 806 printf("bus_dmamem_alloc failed to align memory properly.\n"); 807 } 808 if (map->flags & DMAMAP_COHERENT) 809 atomic_add_32(&maps_coherent, 1); 810 atomic_add_32(&maps_dmamem, 1); 811 atomic_add_32(&maps_total, 1); 812 dmat->map_count++; 813 814 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 815 __func__, dmat, dmat->flags, 0); 816 return (0); 817} 818 819/* 820 * Free a piece of memory and it's allociated dmamap, that was allocated 821 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 822 */ 823void 824bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 825{ 826 struct busdma_bufzone *bufzone; 827 busdma_bufalloc_t ba; 828 829 if (map->flags & DMAMAP_COHERENT) 830 ba = coherent_allocator; 831 else 832 ba = standard_allocator; 833 834 /* Be careful not to access map from here on. */ 835 836 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 837 838 if (bufzone != NULL && dmat->alignment <= bufzone->size && 839 !exclusion_bounce(dmat)) 840 uma_zfree(bufzone->umazone, vaddr); 841 else 842 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 843 844 dmat->map_count--; 845 if (map->flags & DMAMAP_COHERENT) 846 atomic_subtract_32(&maps_coherent, 1); 847 atomic_subtract_32(&maps_total, 1); 848 atomic_subtract_32(&maps_dmamem, 1); 849 free(map, M_DEVBUF); 850 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 851} 852 853static void 854_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 855 bus_size_t buflen, int flags) 856{ 857 bus_addr_t curaddr; 858 bus_size_t sgsize; 859 860 if (map->pagesneeded == 0) { 861 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 862 " map= %p, pagesneeded= %d", 863 dmat->lowaddr, dmat->boundary, dmat->alignment, 864 map, map->pagesneeded); 865 /* 866 * Count the number of bounce pages 867 * needed in order to complete this transfer 868 */ 869 curaddr = buf; 870 while (buflen != 0) { 871 sgsize = MIN(buflen, dmat->maxsegsz); 872 if (must_bounce(dmat, map, curaddr, sgsize) != 0) { 873 sgsize = MIN(sgsize, PAGE_SIZE); 874 map->pagesneeded++; 875 } 876 curaddr += sgsize; 877 buflen -= sgsize; 878 } 879 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 880 } 881} 882 883static void 884_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 885 void *buf, bus_size_t buflen, int flags) 886{ 887 vm_offset_t vaddr; 888 vm_offset_t vendaddr; 889 bus_addr_t paddr; 890 891 if (map->pagesneeded == 0) { 892 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 893 " map= %p, pagesneeded= %d", 894 dmat->lowaddr, dmat->boundary, dmat->alignment, 895 map, map->pagesneeded); 896 /* 897 * Count the number of bounce pages 898 * needed in order to complete this transfer 899 */ 900 vaddr = (vm_offset_t)buf; 901 vendaddr = (vm_offset_t)buf + buflen; 902 903 while (vaddr < vendaddr) { 904 if (__predict_true(map->pmap == kernel_pmap)) 905 paddr = pmap_kextract(vaddr); 906 else 907 paddr = pmap_extract(map->pmap, vaddr); 908 if (must_bounce(dmat, map, paddr, 909 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 910 PAGE_MASK)))) != 0) { 911 map->pagesneeded++; 912 } 913 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); 914 915 } 916 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 917 } 918} 919 920static int 921_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 922{ 923 924 /* Reserve Necessary Bounce Pages */ 925 mtx_lock(&bounce_lock); 926 if (flags & BUS_DMA_NOWAIT) { 927 if (reserve_bounce_pages(dmat, map, 0) != 0) { 928 map->pagesneeded = 0; 929 mtx_unlock(&bounce_lock); 930 return (ENOMEM); 931 } 932 } else { 933 if (reserve_bounce_pages(dmat, map, 1) != 0) { 934 /* Queue us for resources */ 935 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 936 mtx_unlock(&bounce_lock); 937 return (EINPROGRESS); 938 } 939 } 940 mtx_unlock(&bounce_lock); 941 942 return (0); 943} 944 945/* 946 * Add a single contiguous physical range to the segment list. 947 */ 948static int 949_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 950 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 951{ 952 bus_addr_t baddr, bmask; 953 int seg; 954 955 /* 956 * Make sure we don't cross any boundaries. 957 */ 958 bmask = ~(dmat->boundary - 1); 959 if (dmat->boundary > 0) { 960 baddr = (curaddr + dmat->boundary) & bmask; 961 if (sgsize > (baddr - curaddr)) 962 sgsize = (baddr - curaddr); 963 } 964 965 if (dmat->ranges) { 966 struct arm32_dma_range *dr; 967 968 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 969 curaddr); 970 if (dr == NULL) { 971 _bus_dmamap_unload(dmat, map); 972 return (0); 973 } 974 /* 975 * In a valid DMA range. Translate the physical 976 * memory address to an address in the DMA window. 977 */ 978 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 979 } 980 981 /* 982 * Insert chunk into a segment, coalescing with 983 * previous segment if possible. 984 */ 985 seg = *segp; 986 if (seg == -1) { 987 seg = 0; 988 segs[seg].ds_addr = curaddr; 989 segs[seg].ds_len = sgsize; 990 } else { 991 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 992 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 993 (dmat->boundary == 0 || 994 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 995 segs[seg].ds_len += sgsize; 996 else { 997 if (++seg >= dmat->nsegments) 998 return (0); 999 segs[seg].ds_addr = curaddr; 1000 segs[seg].ds_len = sgsize; 1001 } 1002 } 1003 *segp = seg; 1004 return (sgsize); 1005} 1006 1007/* 1008 * Utility function to load a physical buffer. segp contains 1009 * the starting segment on entrace, and the ending segment on exit. 1010 */ 1011int 1012_bus_dmamap_load_phys(bus_dma_tag_t dmat, 1013 bus_dmamap_t map, 1014 vm_paddr_t buf, bus_size_t buflen, 1015 int flags, 1016 bus_dma_segment_t *segs, 1017 int *segp) 1018{ 1019 bus_addr_t curaddr; 1020 bus_size_t sgsize; 1021 int error; 1022 1023 if (segs == NULL) 1024 segs = map->segments; 1025 1026 maploads_total++; 1027 maploads_physmem++; 1028 1029 if (might_bounce(dmat, map, buflen, buflen)) { 1030 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 1031 if (map->pagesneeded != 0) { 1032 maploads_bounced++; 1033 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1034 if (error) 1035 return (error); 1036 } 1037 } 1038 1039 while (buflen > 0) { 1040 curaddr = buf; 1041 sgsize = MIN(buflen, dmat->maxsegsz); 1042 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, 1043 sgsize)) { 1044 sgsize = MIN(sgsize, PAGE_SIZE); 1045 curaddr = add_bounce_page(dmat, map, 0, curaddr, 1046 sgsize); 1047 } 1048 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1049 segp); 1050 if (sgsize == 0) 1051 break; 1052 buf += sgsize; 1053 buflen -= sgsize; 1054 } 1055 1056 /* 1057 * Did we fit? 1058 */ 1059 if (buflen != 0) { 1060 _bus_dmamap_unload(dmat, map); 1061 return (EFBIG); /* XXX better return value here? */ 1062 } 1063 return (0); 1064} 1065 1066int 1067_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1068 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1069 bus_dma_segment_t *segs, int *segp) 1070{ 1071 1072 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1073 segs, segp)); 1074} 1075 1076/* 1077 * Utility function to load a linear buffer. segp contains 1078 * the starting segment on entrace, and the ending segment on exit. 1079 */ 1080int 1081_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 1082 bus_dmamap_t map, 1083 void *buf, bus_size_t buflen, 1084 pmap_t pmap, 1085 int flags, 1086 bus_dma_segment_t *segs, 1087 int *segp) 1088{ 1089 bus_size_t sgsize; 1090 bus_addr_t curaddr; 1091 vm_offset_t vaddr; 1092 struct sync_list *sl; 1093 int error; 1094 1095 maploads_total++; 1096 if (map->flags & DMAMAP_COHERENT) 1097 maploads_coherent++; 1098 if (map->flags & DMAMAP_DMAMEM_ALLOC) 1099 maploads_dmamem++; 1100 1101 if (segs == NULL) 1102 segs = map->segments; 1103 1104 if (flags & BUS_DMA_LOAD_MBUF) { 1105 maploads_mbuf++; 1106 map->flags |= DMAMAP_MBUF; 1107 } 1108 1109 map->pmap = pmap; 1110 1111 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 1112 _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); 1113 if (map->pagesneeded != 0) { 1114 maploads_bounced++; 1115 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1116 if (error) 1117 return (error); 1118 } 1119 } 1120 1121 sl = NULL; 1122 vaddr = (vm_offset_t)buf; 1123 1124 while (buflen > 0) { 1125 /* 1126 * Get the physical address for this segment. 1127 */ 1128 if (__predict_true(map->pmap == kernel_pmap)) 1129 curaddr = pmap_kextract(vaddr); 1130 else 1131 curaddr = pmap_extract(map->pmap, vaddr); 1132 1133 /* 1134 * Compute the segment size, and adjust counts. 1135 */ 1136 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 1137 if (sgsize > dmat->maxsegsz) 1138 sgsize = dmat->maxsegsz; 1139 if (buflen < sgsize) 1140 sgsize = buflen; 1141 1142 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, 1143 sgsize)) { 1144 curaddr = add_bounce_page(dmat, map, vaddr, curaddr, 1145 sgsize); 1146 } else { 1147 sl = &map->slist[map->sync_count - 1]; 1148 if (map->sync_count == 0 || 1149#ifdef ARM_L2_PIPT 1150 curaddr != sl->busaddr + sl->datacount || 1151#endif 1152 vaddr != sl->vaddr + sl->datacount) { 1153 if (++map->sync_count > dmat->nsegments) 1154 goto cleanup; 1155 sl++; 1156 sl->vaddr = vaddr; 1157 sl->datacount = sgsize; 1158 sl->busaddr = curaddr; 1159 } else 1160 sl->datacount += sgsize; 1161 } 1162 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1163 segp); 1164 if (sgsize == 0) 1165 break; 1166 vaddr += sgsize; 1167 buflen -= sgsize; 1168 } 1169 1170cleanup: 1171 /* 1172 * Did we fit? 1173 */ 1174 if (buflen != 0) { 1175 _bus_dmamap_unload(dmat, map); 1176 return (EFBIG); /* XXX better return value here? */ 1177 } 1178 return (0); 1179} 1180 1181 1182void 1183__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 1184 struct memdesc *mem, bus_dmamap_callback_t *callback, 1185 void *callback_arg) 1186{ 1187 1188 map->mem = *mem; 1189 map->dmat = dmat; 1190 map->callback = callback; 1191 map->callback_arg = callback_arg; 1192} 1193 1194bus_dma_segment_t * 1195_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1196 bus_dma_segment_t *segs, int nsegs, int error) 1197{ 1198 1199 if (segs == NULL) 1200 segs = map->segments; 1201 return (segs); 1202} 1203 1204/* 1205 * Release the mapping held by map. 1206 */ 1207void 1208_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1209{ 1210 struct bounce_page *bpage; 1211 struct bounce_zone *bz; 1212 1213 if ((bz = dmat->bounce_zone) != NULL) { 1214 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1215 STAILQ_REMOVE_HEAD(&map->bpages, links); 1216 free_bounce_page(dmat, bpage); 1217 } 1218 1219 bz = dmat->bounce_zone; 1220 bz->free_bpages += map->pagesreserved; 1221 bz->reserved_bpages -= map->pagesreserved; 1222 map->pagesreserved = 0; 1223 map->pagesneeded = 0; 1224 } 1225 map->sync_count = 0; 1226 map->flags &= ~DMAMAP_MBUF; 1227} 1228 1229#ifdef notyetbounceuser 1230 /* If busdma uses user pages, then the interrupt handler could 1231 * be use the kernel vm mapping. Both bounce pages and sync list 1232 * do not cross page boundaries. 1233 * Below is a rough sequence that a person would do to fix the 1234 * user page reference in the kernel vmspace. This would be 1235 * done in the dma post routine. 1236 */ 1237void 1238_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len, 1239 pmap_t pmap, int op) 1240{ 1241 bus_size_t sgsize; 1242 bus_addr_t curaddr; 1243 vm_offset_t va; 1244 1245 /* each synclist entry is contained within a single page. 1246 * 1247 * this would be needed if BUS_DMASYNC_POSTxxxx was implemented 1248 */ 1249 curaddr = pmap_extract(pmap, buf); 1250 va = pmap_dma_map(curaddr); 1251 switch (op) { 1252 case SYNC_USER_INV: 1253 cpu_dcache_wb_range(va, sgsize); 1254 break; 1255 1256 case SYNC_USER_COPYTO: 1257 bcopy((void *)va, (void *)bounce, sgsize); 1258 break; 1259 1260 case SYNC_USER_COPYFROM: 1261 bcopy((void *) bounce, (void *)va, sgsize); 1262 break; 1263 1264 default: 1265 break; 1266 } 1267 1268 pmap_dma_unmap(va); 1269} 1270#endif 1271 1272#ifdef ARM_L2_PIPT 1273#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size) 1274#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size) 1275#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size) 1276#else 1277#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size) 1278#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size) 1279#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size) 1280#endif 1281 1282void 1283_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1284{ 1285 struct bounce_page *bpage; 1286 struct sync_list *sl, *end; 1287 /* 1288 * If the buffer was from user space, it is possible that this is not 1289 * the same vm map, especially on a POST operation. It's not clear that 1290 * dma on userland buffers can work at all right now, certainly not if a 1291 * partial cacheline flush has to be handled. To be safe, until we're 1292 * able to test direct userland dma, panic on a map mismatch. 1293 */ 1294 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1295 if (!pmap_dmap_iscurrent(map->pmap)) 1296 panic("_bus_dmamap_sync: wrong user map for bounce sync."); 1297 /* Handle data bouncing. */ 1298 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1299 "performing bounce", __func__, dmat, dmat->flags, op); 1300 1301 if (op & BUS_DMASYNC_PREWRITE) { 1302 while (bpage != NULL) { 1303 if (bpage->datavaddr != 0) 1304 bcopy((void *)bpage->datavaddr, 1305 (void *)bpage->vaddr, 1306 bpage->datacount); 1307 else 1308 physcopyout(bpage->dataaddr, 1309 (void *)bpage->vaddr, 1310 bpage->datacount); 1311 cpu_dcache_wb_range((vm_offset_t)bpage->vaddr, 1312 bpage->datacount); 1313 l2cache_wb_range((vm_offset_t)bpage->vaddr, 1314 (vm_offset_t)bpage->busaddr, 1315 bpage->datacount); 1316 bpage = STAILQ_NEXT(bpage, links); 1317 } 1318 dmat->bounce_zone->total_bounced++; 1319 } 1320 1321 if (op & BUS_DMASYNC_PREREAD) { 1322 bpage = STAILQ_FIRST(&map->bpages); 1323 while (bpage != NULL) { 1324 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr, 1325 bpage->datacount); 1326 l2cache_inv_range((vm_offset_t)bpage->vaddr, 1327 (vm_offset_t)bpage->busaddr, 1328 bpage->datacount); 1329 bpage = STAILQ_NEXT(bpage, links); 1330 } 1331 } 1332 if (op & BUS_DMASYNC_POSTREAD) { 1333 while (bpage != NULL) { 1334 vm_offset_t startv; 1335 vm_paddr_t startp; 1336 int len; 1337 1338 startv = bpage->vaddr &~ arm_dcache_align_mask; 1339 startp = bpage->busaddr &~ arm_dcache_align_mask; 1340 len = bpage->datacount; 1341 1342 if (startv != bpage->vaddr) 1343 len += bpage->vaddr & arm_dcache_align_mask; 1344 if (len & arm_dcache_align_mask) 1345 len = (len - 1346 (len & arm_dcache_align_mask)) + 1347 arm_dcache_align; 1348 cpu_dcache_inv_range(startv, len); 1349 l2cache_inv_range(startv, startp, len); 1350 if (bpage->datavaddr != 0) 1351 bcopy((void *)bpage->vaddr, 1352 (void *)bpage->datavaddr, 1353 bpage->datacount); 1354 else 1355 physcopyin((void *)bpage->vaddr, 1356 bpage->dataaddr, 1357 bpage->datacount); 1358 bpage = STAILQ_NEXT(bpage, links); 1359 } 1360 dmat->bounce_zone->total_bounced++; 1361 } 1362 } 1363 if (map->flags & DMAMAP_COHERENT) 1364 return; 1365 1366 if (map->sync_count != 0) { 1367 if (!pmap_dmap_iscurrent(map->pmap)) 1368 panic("_bus_dmamap_sync: wrong user map for sync."); 1369 /* ARM caches are not self-snooping for dma */ 1370 1371 sl = &map->slist[0]; 1372 end = &map->slist[map->sync_count]; 1373 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1374 "performing sync", __func__, dmat, dmat->flags, op); 1375 1376 switch (op) { 1377 case BUS_DMASYNC_PREWRITE: 1378 while (sl != end) { 1379 cpu_dcache_wb_range(sl->vaddr, sl->datacount); 1380 l2cache_wb_range(sl->vaddr, sl->busaddr, 1381 sl->datacount); 1382 sl++; 1383 } 1384 break; 1385 1386 case BUS_DMASYNC_PREREAD: 1387 while (sl != end) { 1388 cpu_dcache_inv_range(sl->vaddr, sl->datacount); 1389 l2cache_inv_range(sl->vaddr, sl->busaddr, 1390 sl->datacount); 1391 sl++; 1392 } 1393 break; 1394 1395 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1396 while (sl != end) { 1397 cpu_dcache_wbinv_range(sl->vaddr, sl->datacount); 1398 l2cache_wbinv_range(sl->vaddr, 1399 sl->busaddr, sl->datacount); 1400 sl++; 1401 } 1402 break; 1403 1404 case BUS_DMASYNC_POSTREAD: 1405 case BUS_DMASYNC_POSTWRITE: 1406 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1407 break; 1408 default: 1409 panic("unsupported combination of sync operations: 0x%08x\n", op); 1410 break; 1411 } 1412 } 1413} 1414 1415static void 1416init_bounce_pages(void *dummy __unused) 1417{ 1418 1419 total_bpages = 0; 1420 STAILQ_INIT(&bounce_zone_list); 1421 STAILQ_INIT(&bounce_map_waitinglist); 1422 STAILQ_INIT(&bounce_map_callbacklist); 1423 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1424} 1425SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1426 1427static struct sysctl_ctx_list * 1428busdma_sysctl_tree(struct bounce_zone *bz) 1429{ 1430 return (&bz->sysctl_tree); 1431} 1432 1433static struct sysctl_oid * 1434busdma_sysctl_tree_top(struct bounce_zone *bz) 1435{ 1436 return (bz->sysctl_tree_top); 1437} 1438 1439static int 1440alloc_bounce_zone(bus_dma_tag_t dmat) 1441{ 1442 struct bounce_zone *bz; 1443 1444 /* Check to see if we already have a suitable zone */ 1445 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1446 if ((dmat->alignment <= bz->alignment) && 1447 (dmat->lowaddr >= bz->lowaddr)) { 1448 dmat->bounce_zone = bz; 1449 return (0); 1450 } 1451 } 1452 1453 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1454 M_NOWAIT | M_ZERO)) == NULL) 1455 return (ENOMEM); 1456 1457 STAILQ_INIT(&bz->bounce_page_list); 1458 bz->free_bpages = 0; 1459 bz->reserved_bpages = 0; 1460 bz->active_bpages = 0; 1461 bz->lowaddr = dmat->lowaddr; 1462 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1463 bz->map_count = 0; 1464 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1465 busdma_zonecount++; 1466 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1467 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1468 dmat->bounce_zone = bz; 1469 1470 sysctl_ctx_init(&bz->sysctl_tree); 1471 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1472 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1473 CTLFLAG_RD, 0, ""); 1474 if (bz->sysctl_tree_top == NULL) { 1475 sysctl_ctx_free(&bz->sysctl_tree); 1476 return (0); /* XXX error code? */ 1477 } 1478 1479 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1480 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1481 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1482 "Total bounce pages"); 1483 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1484 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1485 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1486 "Free bounce pages"); 1487 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1488 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1489 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1490 "Reserved bounce pages"); 1491 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1492 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1493 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1494 "Active bounce pages"); 1495 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1496 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1497 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1498 "Total bounce requests (pages bounced)"); 1499 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1500 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1501 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1502 "Total bounce requests that were deferred"); 1503 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1504 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1505 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1506 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1507 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1508 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1509 1510 return (0); 1511} 1512 1513static int 1514alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1515{ 1516 struct bounce_zone *bz; 1517 int count; 1518 1519 bz = dmat->bounce_zone; 1520 count = 0; 1521 while (numpages > 0) { 1522 struct bounce_page *bpage; 1523 1524 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1525 M_NOWAIT | M_ZERO); 1526 1527 if (bpage == NULL) 1528 break; 1529 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1530 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1531 if (bpage->vaddr == 0) { 1532 free(bpage, M_DEVBUF); 1533 break; 1534 } 1535 bpage->busaddr = pmap_kextract(bpage->vaddr); 1536 mtx_lock(&bounce_lock); 1537 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1538 total_bpages++; 1539 bz->total_bpages++; 1540 bz->free_bpages++; 1541 mtx_unlock(&bounce_lock); 1542 count++; 1543 numpages--; 1544 } 1545 return (count); 1546} 1547 1548static int 1549reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1550{ 1551 struct bounce_zone *bz; 1552 int pages; 1553 1554 mtx_assert(&bounce_lock, MA_OWNED); 1555 bz = dmat->bounce_zone; 1556 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1557 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1558 return (map->pagesneeded - (map->pagesreserved + pages)); 1559 bz->free_bpages -= pages; 1560 bz->reserved_bpages += pages; 1561 map->pagesreserved += pages; 1562 pages = map->pagesneeded - map->pagesreserved; 1563 1564 return (pages); 1565} 1566 1567static bus_addr_t 1568add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1569 bus_addr_t addr, bus_size_t size) 1570{ 1571 struct bounce_zone *bz; 1572 struct bounce_page *bpage; 1573 1574 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1575 KASSERT(map != NULL, 1576 ("add_bounce_page: bad map %p", map)); 1577 1578 bz = dmat->bounce_zone; 1579 if (map->pagesneeded == 0) 1580 panic("add_bounce_page: map doesn't need any pages"); 1581 map->pagesneeded--; 1582 1583 if (map->pagesreserved == 0) 1584 panic("add_bounce_page: map doesn't need any pages"); 1585 map->pagesreserved--; 1586 1587 mtx_lock(&bounce_lock); 1588 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1589 if (bpage == NULL) 1590 panic("add_bounce_page: free page list is empty"); 1591 1592 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1593 bz->reserved_bpages--; 1594 bz->active_bpages++; 1595 mtx_unlock(&bounce_lock); 1596 1597 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1598 /* Page offset needs to be preserved. */ 1599 bpage->vaddr |= vaddr & PAGE_MASK; 1600 bpage->busaddr |= vaddr & PAGE_MASK; 1601 } 1602 bpage->datavaddr = vaddr; 1603 bpage->dataaddr = addr; 1604 bpage->datacount = size; 1605 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1606 return (bpage->busaddr); 1607} 1608 1609static void 1610free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1611{ 1612 struct bus_dmamap *map; 1613 struct bounce_zone *bz; 1614 1615 bz = dmat->bounce_zone; 1616 bpage->datavaddr = 0; 1617 bpage->datacount = 0; 1618 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1619 /* 1620 * Reset the bounce page to start at offset 0. Other uses 1621 * of this bounce page may need to store a full page of 1622 * data and/or assume it starts on a page boundary. 1623 */ 1624 bpage->vaddr &= ~PAGE_MASK; 1625 bpage->busaddr &= ~PAGE_MASK; 1626 } 1627 1628 mtx_lock(&bounce_lock); 1629 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1630 bz->free_bpages++; 1631 bz->active_bpages--; 1632 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1633 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1634 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1635 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1636 map, links); 1637 busdma_swi_pending = 1; 1638 bz->total_deferred++; 1639 swi_sched(vm_ih, 0); 1640 } 1641 } 1642 mtx_unlock(&bounce_lock); 1643} 1644 1645void 1646busdma_swi(void) 1647{ 1648 bus_dma_tag_t dmat; 1649 struct bus_dmamap *map; 1650 1651 mtx_lock(&bounce_lock); 1652 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1653 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1654 mtx_unlock(&bounce_lock); 1655 dmat = map->dmat; 1656 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1657 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1658 map->callback_arg, BUS_DMA_WAITOK); 1659 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1660 mtx_lock(&bounce_lock); 1661 } 1662 mtx_unlock(&bounce_lock); 1663} 1664