busdma_machdep-v6.c revision 330897
1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012-2015 Ian Lepore 5 * Copyright (c) 2010 Mark Tinguely 6 * Copyright (c) 2004 Olivier Houchard 7 * Copyright (c) 2002 Peter Grehan 8 * Copyright (c) 1997, 1998 Justin T. Gibbs. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions, and the following disclaimer, 16 * without modification, immediately at the beginning of the file. 17 * 2. The name of the author may not be used to endorse or promote products 18 * derived from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb 33 */ 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD: stable/11/sys/arm/arm/busdma_machdep-v6.c 330897 2018-03-14 03:19:51Z eadler $"); 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/malloc.h> 41#include <sys/bus.h> 42#include <sys/busdma_bufalloc.h> 43#include <sys/counter.h> 44#include <sys/interrupt.h> 45#include <sys/kernel.h> 46#include <sys/ktr.h> 47#include <sys/lock.h> 48#include <sys/memdesc.h> 49#include <sys/proc.h> 50#include <sys/mutex.h> 51#include <sys/sysctl.h> 52#include <sys/uio.h> 53 54#include <vm/vm.h> 55#include <vm/vm_page.h> 56#include <vm/vm_map.h> 57#include <vm/vm_extern.h> 58#include <vm/vm_kern.h> 59 60#include <machine/atomic.h> 61#include <machine/bus.h> 62#include <machine/cpu-v6.h> 63#include <machine/md_var.h> 64 65#define BUSDMA_DCACHE_ALIGN cpuinfo.dcache_line_size 66#define BUSDMA_DCACHE_MASK cpuinfo.dcache_line_mask 67 68#define MAX_BPAGES 64 69#define MAX_DMA_SEGMENTS 4096 70#define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2 71#define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3 72#define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE) 73#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 74 75struct bounce_zone; 76 77struct bus_dma_tag { 78 bus_dma_tag_t parent; 79 bus_size_t alignment; 80 bus_addr_t boundary; 81 bus_addr_t lowaddr; 82 bus_addr_t highaddr; 83 bus_dma_filter_t *filter; 84 void *filterarg; 85 bus_size_t maxsize; 86 u_int nsegments; 87 bus_size_t maxsegsz; 88 int flags; 89 int ref_count; 90 int map_count; 91 bus_dma_lock_t *lockfunc; 92 void *lockfuncarg; 93 struct bounce_zone *bounce_zone; 94}; 95 96struct bounce_page { 97 vm_offset_t vaddr; /* kva of bounce buffer */ 98 bus_addr_t busaddr; /* Physical address */ 99 vm_offset_t datavaddr; /* kva of client data */ 100 vm_page_t datapage; /* physical page of client data */ 101 vm_offset_t dataoffs; /* page offset of client data */ 102 bus_size_t datacount; /* client data count */ 103 STAILQ_ENTRY(bounce_page) links; 104}; 105 106struct sync_list { 107 vm_offset_t vaddr; /* kva of client data */ 108 bus_addr_t paddr; /* physical address */ 109 vm_page_t pages; /* starting page of client data */ 110 bus_size_t datacount; /* client data count */ 111}; 112 113int busdma_swi_pending; 114 115struct bounce_zone { 116 STAILQ_ENTRY(bounce_zone) links; 117 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 118 int total_bpages; 119 int free_bpages; 120 int reserved_bpages; 121 int active_bpages; 122 int total_bounced; 123 int total_deferred; 124 int map_count; 125 bus_size_t alignment; 126 bus_addr_t lowaddr; 127 char zoneid[8]; 128 char lowaddrid[20]; 129 struct sysctl_ctx_list sysctl_tree; 130 struct sysctl_oid *sysctl_tree_top; 131}; 132 133static struct mtx bounce_lock; 134static int total_bpages; 135static int busdma_zonecount; 136static uint32_t tags_total; 137static uint32_t maps_total; 138static uint32_t maps_dmamem; 139static uint32_t maps_coherent; 140static counter_u64_t maploads_total; 141static counter_u64_t maploads_bounced; 142static counter_u64_t maploads_coherent; 143static counter_u64_t maploads_dmamem; 144static counter_u64_t maploads_mbuf; 145static counter_u64_t maploads_physmem; 146 147static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 148 149SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 150SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0, 151 "Number of active tags"); 152SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0, 153 "Number of active maps"); 154SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0, 155 "Number of active maps for bus_dmamem_alloc buffers"); 156SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0, 157 "Number of active maps with BUS_DMA_COHERENT flag set"); 158SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, 159 &maploads_total, "Number of load operations performed"); 160SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, 161 &maploads_bounced, "Number of load operations that used bounce buffers"); 162SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, 163 &maploads_dmamem, "Number of load operations on BUS_DMA_COHERENT memory"); 164SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, 165 &maploads_dmamem, "Number of load operations on bus_dmamem_alloc buffers"); 166SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, 167 &maploads_mbuf, "Number of load operations for mbufs"); 168SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, 169 &maploads_physmem, "Number of load operations on physical buffers"); 170SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 171 "Total bounce pages"); 172 173struct bus_dmamap { 174 struct bp_list bpages; 175 int pagesneeded; 176 int pagesreserved; 177 bus_dma_tag_t dmat; 178 struct memdesc mem; 179 bus_dmamap_callback_t *callback; 180 void *callback_arg; 181 int flags; 182#define DMAMAP_COHERENT (1 << 0) 183#define DMAMAP_DMAMEM_ALLOC (1 << 1) 184#define DMAMAP_MBUF (1 << 2) 185 STAILQ_ENTRY(bus_dmamap) links; 186 bus_dma_segment_t *segments; 187 int sync_count; 188 struct sync_list slist[]; 189}; 190 191static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 192static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 193 194static void init_bounce_pages(void *dummy); 195static int alloc_bounce_zone(bus_dma_tag_t dmat); 196static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 197static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 198 int commit); 199static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 200 vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 201static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 202static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, 203 bus_dmamap_t map, void *buf, bus_size_t buflen, int flags); 204static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 205 vm_paddr_t buf, bus_size_t buflen, int flags); 206static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 207 int flags); 208static void dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size); 209static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op); 210 211static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 212static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 213 214MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 215MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages"); 216 217static void 218busdma_init(void *dummy) 219{ 220 int uma_flags; 221 222 maploads_total = counter_u64_alloc(M_WAITOK); 223 maploads_bounced = counter_u64_alloc(M_WAITOK); 224 maploads_coherent = counter_u64_alloc(M_WAITOK); 225 maploads_dmamem = counter_u64_alloc(M_WAITOK); 226 maploads_mbuf = counter_u64_alloc(M_WAITOK); 227 maploads_physmem = counter_u64_alloc(M_WAITOK); 228 229 uma_flags = 0; 230 231 /* Create a cache of buffers in standard (cacheable) memory. */ 232 standard_allocator = busdma_bufalloc_create("buffer", 233 BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ 234 NULL, /* uma_alloc func */ 235 NULL, /* uma_free func */ 236 uma_flags); /* uma_zcreate_flags */ 237 238#ifdef INVARIANTS 239 /* 240 * Force UMA zone to allocate service structures like 241 * slabs using own allocator. uma_debug code performs 242 * atomic ops on uma_slab_t fields and safety of this 243 * operation is not guaranteed for write-back caches 244 */ 245 uma_flags = UMA_ZONE_OFFPAGE; 246#endif 247 /* 248 * Create a cache of buffers in uncacheable memory, to implement the 249 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 250 */ 251 coherent_allocator = busdma_bufalloc_create("coherent", 252 BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ 253 busdma_bufalloc_alloc_uncacheable, 254 busdma_bufalloc_free_uncacheable, 255 uma_flags); /* uma_zcreate_flags */ 256} 257 258/* 259 * This init historically used SI_SUB_VM, but now the init code requires 260 * malloc(9) using M_BUSDMA memory and the pcpu zones for counter(9), which get 261 * set up by SI_SUB_KMEM and SI_ORDER_LAST, so we'll go right after that by 262 * using SI_SUB_KMEM+1. 263 */ 264SYSINIT(busdma, SI_SUB_KMEM+1, SI_ORDER_FIRST, busdma_init, NULL); 265 266/* 267 * This routine checks the exclusion zone constraints from a tag against the 268 * physical RAM available on the machine. If a tag specifies an exclusion zone 269 * but there's no RAM in that zone, then we avoid allocating resources to bounce 270 * a request, and we can use any memory allocator (as opposed to needing 271 * kmem_alloc_contig() just because it can allocate pages in an address range). 272 * 273 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 274 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 275 * possibly have RAM at an address higher than the highest address we can 276 * express, so we take a fast out. 277 */ 278static int 279exclusion_bounce_check(vm_offset_t lowaddr, vm_offset_t highaddr) 280{ 281 int i; 282 283 if (lowaddr >= BUS_SPACE_MAXADDR) 284 return (0); 285 286 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 287 if ((lowaddr >= phys_avail[i] && lowaddr < phys_avail[i + 1]) || 288 (lowaddr < phys_avail[i] && highaddr >= phys_avail[i])) 289 return (1); 290 } 291 return (0); 292} 293 294/* 295 * Return true if the tag has an exclusion zone that could lead to bouncing. 296 */ 297static __inline int 298exclusion_bounce(bus_dma_tag_t dmat) 299{ 300 301 return (dmat->flags & BUS_DMA_EXCL_BOUNCE); 302} 303 304/* 305 * Return true if the given address does not fall on the alignment boundary. 306 */ 307static __inline int 308alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 309{ 310 311 return (addr & (dmat->alignment - 1)); 312} 313 314/* 315 * Return true if the DMA should bounce because the start or end does not fall 316 * on a cacheline boundary (which would require a partial cacheline flush). 317 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 318 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 319 * strict rule that such memory cannot be accessed by the CPU while DMA is in 320 * progress (or by multiple DMA engines at once), so that it's always safe to do 321 * full cacheline flushes even if that affects memory outside the range of a 322 * given DMA operation that doesn't involve the full allocated buffer. If we're 323 * mapping an mbuf, that follows the same rules as a buffer we allocated. 324 */ 325static __inline int 326cacheline_bounce(bus_dmamap_t map, bus_addr_t addr, bus_size_t size) 327{ 328 329 if (map->flags & (DMAMAP_DMAMEM_ALLOC | DMAMAP_COHERENT | DMAMAP_MBUF)) 330 return (0); 331 return ((addr | size) & BUSDMA_DCACHE_MASK); 332} 333 334/* 335 * Return true if we might need to bounce the DMA described by addr and size. 336 * 337 * This is used to quick-check whether we need to do the more expensive work of 338 * checking the DMA page-by-page looking for alignment and exclusion bounces. 339 * 340 * Note that the addr argument might be either virtual or physical. It doesn't 341 * matter because we only look at the low-order bits, which are the same in both 342 * address spaces. 343 */ 344static __inline int 345might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t addr, 346 bus_size_t size) 347{ 348 349 return ((dmat->flags & BUS_DMA_EXCL_BOUNCE) || 350 alignment_bounce(dmat, addr) || 351 cacheline_bounce(map, addr, size)); 352} 353 354/* 355 * Return true if we must bounce the DMA described by paddr and size. 356 * 357 * Bouncing can be triggered by DMA that doesn't begin and end on cacheline 358 * boundaries, or doesn't begin on an alignment boundary, or falls within the 359 * exclusion zone of any tag in the ancestry chain. 360 * 361 * For exclusions, walk the chain of tags comparing paddr to the exclusion zone 362 * within each tag. If the tag has a filter function, use it to decide whether 363 * the DMA needs to bounce, otherwise any DMA within the zone bounces. 364 */ 365static int 366must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 367 bus_size_t size) 368{ 369 370 if (cacheline_bounce(map, paddr, size)) 371 return (1); 372 373 /* 374 * The tag already contains ancestors' alignment restrictions so this 375 * check doesn't need to be inside the loop. 376 */ 377 if (alignment_bounce(dmat, paddr)) 378 return (1); 379 380 /* 381 * Even though each tag has an exclusion zone that is a superset of its 382 * own and all its ancestors' exclusions, the exclusion zone of each tag 383 * up the chain must be checked within the loop, because the busdma 384 * rules say the filter function is called only when the address lies 385 * within the low-highaddr range of the tag that filterfunc belongs to. 386 */ 387 while (dmat != NULL && exclusion_bounce(dmat)) { 388 if ((paddr >= dmat->lowaddr && paddr <= dmat->highaddr) && 389 (dmat->filter == NULL || 390 dmat->filter(dmat->filterarg, paddr) != 0)) 391 return (1); 392 dmat = dmat->parent; 393 } 394 395 return (0); 396} 397 398/* 399 * Convenience function for manipulating driver locks from busdma (during 400 * busdma_swi, for example). Drivers that don't provide their own locks 401 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 402 * non-mutex locking scheme don't have to use this at all. 403 */ 404void 405busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 406{ 407 struct mtx *dmtx; 408 409 dmtx = (struct mtx *)arg; 410 switch (op) { 411 case BUS_DMA_LOCK: 412 mtx_lock(dmtx); 413 break; 414 case BUS_DMA_UNLOCK: 415 mtx_unlock(dmtx); 416 break; 417 default: 418 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 419 } 420} 421 422/* 423 * dflt_lock should never get called. It gets put into the dma tag when 424 * lockfunc == NULL, which is only valid if the maps that are associated 425 * with the tag are meant to never be defered. 426 * XXX Should have a way to identify which driver is responsible here. 427 */ 428static void 429dflt_lock(void *arg, bus_dma_lock_op_t op) 430{ 431 432 panic("driver error: busdma dflt_lock called"); 433} 434 435/* 436 * Allocate a device specific dma_tag. 437 */ 438int 439bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 440 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 441 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 442 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 443 void *lockfuncarg, bus_dma_tag_t *dmat) 444{ 445 bus_dma_tag_t newtag; 446 int error = 0; 447 448 /* Basic sanity checking. */ 449 KASSERT(boundary == 0 || powerof2(boundary), 450 ("dma tag boundary %lu, must be a power of 2", boundary)); 451 KASSERT(boundary == 0 || boundary >= maxsegsz, 452 ("dma tag boundary %lu is < maxsegsz %lu\n", boundary, maxsegsz)); 453 KASSERT(alignment != 0 && powerof2(alignment), 454 ("dma tag alignment %lu, must be non-zero power of 2", alignment)); 455 KASSERT(maxsegsz != 0, ("dma tag maxsegsz must not be zero")); 456 457 /* Return a NULL tag on failure */ 458 *dmat = NULL; 459 460 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, 461 M_ZERO | M_NOWAIT); 462 if (newtag == NULL) { 463 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 464 __func__, newtag, 0, error); 465 return (ENOMEM); 466 } 467 468 newtag->parent = parent; 469 newtag->alignment = alignment; 470 newtag->boundary = boundary; 471 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 472 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 473 (PAGE_SIZE - 1); 474 newtag->filter = filter; 475 newtag->filterarg = filterarg; 476 newtag->maxsize = maxsize; 477 newtag->nsegments = nsegments; 478 newtag->maxsegsz = maxsegsz; 479 newtag->flags = flags; 480 newtag->ref_count = 1; /* Count ourself */ 481 newtag->map_count = 0; 482 if (lockfunc != NULL) { 483 newtag->lockfunc = lockfunc; 484 newtag->lockfuncarg = lockfuncarg; 485 } else { 486 newtag->lockfunc = dflt_lock; 487 newtag->lockfuncarg = NULL; 488 } 489 490 /* Take into account any restrictions imposed by our parent tag */ 491 if (parent != NULL) { 492 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 493 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 494 newtag->alignment = MAX(parent->alignment, newtag->alignment); 495 newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; 496 if (newtag->boundary == 0) 497 newtag->boundary = parent->boundary; 498 else if (parent->boundary != 0) 499 newtag->boundary = MIN(parent->boundary, 500 newtag->boundary); 501 if (newtag->filter == NULL) { 502 /* 503 * Short circuit to looking at our parent directly 504 * since we have encapsulated all of its information 505 */ 506 newtag->filter = parent->filter; 507 newtag->filterarg = parent->filterarg; 508 newtag->parent = parent->parent; 509 } 510 if (newtag->parent != NULL) 511 atomic_add_int(&parent->ref_count, 1); 512 } 513 514 if (exclusion_bounce_check(newtag->lowaddr, newtag->highaddr)) 515 newtag->flags |= BUS_DMA_EXCL_BOUNCE; 516 if (alignment_bounce(newtag, 1)) 517 newtag->flags |= BUS_DMA_ALIGN_BOUNCE; 518 519 /* 520 * Any request can auto-bounce due to cacheline alignment, in addition 521 * to any alignment or boundary specifications in the tag, so if the 522 * ALLOCNOW flag is set, there's always work to do. 523 */ 524 if ((flags & BUS_DMA_ALLOCNOW) != 0) { 525 struct bounce_zone *bz; 526 /* 527 * Round size up to a full page, and add one more page because 528 * there can always be one more boundary crossing than the 529 * number of pages in a transfer. 530 */ 531 maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE; 532 533 if ((error = alloc_bounce_zone(newtag)) != 0) { 534 free(newtag, M_BUSDMA); 535 return (error); 536 } 537 bz = newtag->bounce_zone; 538 539 if (ptoa(bz->total_bpages) < maxsize) { 540 int pages; 541 542 pages = atop(maxsize) - bz->total_bpages; 543 544 /* Add pages to our bounce pool */ 545 if (alloc_bounce_pages(newtag, pages) < pages) 546 error = ENOMEM; 547 } 548 /* Performed initial allocation */ 549 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 550 } else 551 newtag->bounce_zone = NULL; 552 553 if (error != 0) { 554 free(newtag, M_BUSDMA); 555 } else { 556 atomic_add_32(&tags_total, 1); 557 *dmat = newtag; 558 } 559 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 560 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 561 return (error); 562} 563 564int 565bus_dma_tag_destroy(bus_dma_tag_t dmat) 566{ 567 bus_dma_tag_t dmat_copy; 568 int error; 569 570 error = 0; 571 dmat_copy = dmat; 572 573 if (dmat != NULL) { 574 575 if (dmat->map_count != 0) { 576 error = EBUSY; 577 goto out; 578 } 579 580 while (dmat != NULL) { 581 bus_dma_tag_t parent; 582 583 parent = dmat->parent; 584 atomic_subtract_int(&dmat->ref_count, 1); 585 if (dmat->ref_count == 0) { 586 atomic_subtract_32(&tags_total, 1); 587 free(dmat, M_BUSDMA); 588 /* 589 * Last reference count, so 590 * release our reference 591 * count on our parent. 592 */ 593 dmat = parent; 594 } else 595 dmat = NULL; 596 } 597 } 598out: 599 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 600 return (error); 601} 602 603static int 604allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp) 605{ 606 struct bounce_zone *bz; 607 int maxpages; 608 int error; 609 610 if (dmat->bounce_zone == NULL) 611 if ((error = alloc_bounce_zone(dmat)) != 0) 612 return (error); 613 bz = dmat->bounce_zone; 614 /* Initialize the new map */ 615 STAILQ_INIT(&(mapp->bpages)); 616 617 /* 618 * Attempt to add pages to our pool on a per-instance basis up to a sane 619 * limit. Even if the tag isn't flagged as COULD_BOUNCE due to 620 * alignment and boundary constraints, it could still auto-bounce due to 621 * cacheline alignment, which requires at most two bounce pages. 622 */ 623 if (dmat->flags & BUS_DMA_COULD_BOUNCE) 624 maxpages = MAX_BPAGES; 625 else 626 maxpages = 2 * bz->map_count; 627 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || 628 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 629 int pages; 630 631 pages = atop(roundup2(dmat->maxsize, PAGE_SIZE)) + 1; 632 pages = MIN(maxpages - bz->total_bpages, pages); 633 pages = MAX(pages, 2); 634 if (alloc_bounce_pages(dmat, pages) < pages) 635 return (ENOMEM); 636 637 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) 638 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 639 } 640 bz->map_count++; 641 return (0); 642} 643 644static bus_dmamap_t 645allocate_map(bus_dma_tag_t dmat, int mflags) 646{ 647 int mapsize, segsize; 648 bus_dmamap_t map; 649 650 /* 651 * Allocate the map. The map structure ends with an embedded 652 * variable-sized array of sync_list structures. Following that 653 * we allocate enough extra space to hold the array of bus_dma_segments. 654 */ 655 KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, 656 ("cannot allocate %u dma segments (max is %u)", 657 dmat->nsegments, MAX_DMA_SEGMENTS)); 658 segsize = sizeof(struct bus_dma_segment) * dmat->nsegments; 659 mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments; 660 map = malloc(mapsize + segsize, M_BUSDMA, mflags | M_ZERO); 661 if (map == NULL) { 662 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 663 return (NULL); 664 } 665 map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize); 666 return (map); 667} 668 669/* 670 * Allocate a handle for mapping from kva/uva/physical 671 * address space into bus device space. 672 */ 673int 674bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 675{ 676 bus_dmamap_t map; 677 int error = 0; 678 679 *mapp = map = allocate_map(dmat, M_NOWAIT); 680 if (map == NULL) { 681 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 682 return (ENOMEM); 683 } 684 685 /* 686 * Bouncing might be required if the driver asks for an exclusion 687 * region, a data alignment that is stricter than 1, or DMA that begins 688 * or ends with a partial cacheline. Whether bouncing will actually 689 * happen can't be known until mapping time, but we need to pre-allocate 690 * resources now because we might not be allowed to at mapping time. 691 */ 692 error = allocate_bz_and_pages(dmat, map); 693 if (error != 0) { 694 free(map, M_BUSDMA); 695 *mapp = NULL; 696 return (error); 697 } 698 if (map->flags & DMAMAP_COHERENT) 699 atomic_add_32(&maps_coherent, 1); 700 atomic_add_32(&maps_total, 1); 701 dmat->map_count++; 702 703 return (0); 704} 705 706/* 707 * Destroy a handle for mapping from kva/uva/physical 708 * address space into bus device space. 709 */ 710int 711bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 712{ 713 714 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 715 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 716 __func__, dmat, EBUSY); 717 return (EBUSY); 718 } 719 if (dmat->bounce_zone) 720 dmat->bounce_zone->map_count--; 721 if (map->flags & DMAMAP_COHERENT) 722 atomic_subtract_32(&maps_coherent, 1); 723 atomic_subtract_32(&maps_total, 1); 724 free(map, M_BUSDMA); 725 dmat->map_count--; 726 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 727 return (0); 728} 729 730/* 731 * Allocate a piece of memory that can be efficiently mapped into bus device 732 * space based on the constraints listed in the dma tag. Returns a pointer to 733 * the allocated memory, and a pointer to an associated bus_dmamap. 734 */ 735int 736bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, 737 bus_dmamap_t *mapp) 738{ 739 busdma_bufalloc_t ba; 740 struct busdma_bufzone *bufzone; 741 bus_dmamap_t map; 742 vm_memattr_t memattr; 743 int mflags; 744 745 if (flags & BUS_DMA_NOWAIT) 746 mflags = M_NOWAIT; 747 else 748 mflags = M_WAITOK; 749 if (flags & BUS_DMA_ZERO) 750 mflags |= M_ZERO; 751 752 *mapp = map = allocate_map(dmat, mflags); 753 if (map == NULL) { 754 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 755 __func__, dmat, dmat->flags, ENOMEM); 756 return (ENOMEM); 757 } 758 map->flags = DMAMAP_DMAMEM_ALLOC; 759 760 /* Choose a busdma buffer allocator based on memory type flags. */ 761 if (flags & BUS_DMA_COHERENT) { 762 memattr = VM_MEMATTR_UNCACHEABLE; 763 ba = coherent_allocator; 764 map->flags |= DMAMAP_COHERENT; 765 } else { 766 memattr = VM_MEMATTR_DEFAULT; 767 ba = standard_allocator; 768 } 769 770 /* 771 * Try to find a bufzone in the allocator that holds a cache of buffers 772 * of the right size for this request. If the buffer is too big to be 773 * held in the allocator cache, this returns NULL. 774 */ 775 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 776 777 /* 778 * Allocate the buffer from the uma(9) allocator if... 779 * - It's small enough to be in the allocator (bufzone not NULL). 780 * - The alignment constraint isn't larger than the allocation size 781 * (the allocator aligns buffers to their size boundaries). 782 * - There's no need to handle lowaddr/highaddr exclusion zones. 783 * else allocate non-contiguous pages if... 784 * - The page count that could get allocated doesn't exceed 785 * nsegments also when the maximum segment size is less 786 * than PAGE_SIZE. 787 * - The alignment constraint isn't larger than a page boundary. 788 * - There are no boundary-crossing constraints. 789 * else allocate a block of contiguous pages because one or more of the 790 * constraints is something that only the contig allocator can fulfill. 791 */ 792 if (bufzone != NULL && dmat->alignment <= bufzone->size && 793 !exclusion_bounce(dmat)) { 794 *vaddr = uma_zalloc(bufzone->umazone, mflags); 795 } else if (dmat->nsegments >= 796 howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) && 797 dmat->alignment <= PAGE_SIZE && 798 (dmat->boundary % PAGE_SIZE) == 0) { 799 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 800 mflags, 0, dmat->lowaddr, memattr); 801 } else { 802 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 803 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 804 memattr); 805 } 806 if (*vaddr == NULL) { 807 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 808 __func__, dmat, dmat->flags, ENOMEM); 809 free(map, M_BUSDMA); 810 *mapp = NULL; 811 return (ENOMEM); 812 } 813 if (map->flags & DMAMAP_COHERENT) 814 atomic_add_32(&maps_coherent, 1); 815 atomic_add_32(&maps_dmamem, 1); 816 atomic_add_32(&maps_total, 1); 817 dmat->map_count++; 818 819 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 820 __func__, dmat, dmat->flags, 0); 821 return (0); 822} 823 824/* 825 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 826 * its associated map. 827 */ 828void 829bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 830{ 831 struct busdma_bufzone *bufzone; 832 busdma_bufalloc_t ba; 833 834 if (map->flags & DMAMAP_COHERENT) 835 ba = coherent_allocator; 836 else 837 ba = standard_allocator; 838 839 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 840 841 if (bufzone != NULL && dmat->alignment <= bufzone->size && 842 !exclusion_bounce(dmat)) 843 uma_zfree(bufzone->umazone, vaddr); 844 else 845 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 846 847 dmat->map_count--; 848 if (map->flags & DMAMAP_COHERENT) 849 atomic_subtract_32(&maps_coherent, 1); 850 atomic_subtract_32(&maps_total, 1); 851 atomic_subtract_32(&maps_dmamem, 1); 852 free(map, M_BUSDMA); 853 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 854} 855 856static void 857_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 858 bus_size_t buflen, int flags) 859{ 860 bus_addr_t curaddr; 861 bus_size_t sgsize; 862 863 if (map->pagesneeded == 0) { 864 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 865 " map= %p, pagesneeded= %d", 866 dmat->lowaddr, dmat->boundary, dmat->alignment, 867 map, map->pagesneeded); 868 /* 869 * Count the number of bounce pages 870 * needed in order to complete this transfer 871 */ 872 curaddr = buf; 873 while (buflen != 0) { 874 sgsize = MIN(buflen, dmat->maxsegsz); 875 if (must_bounce(dmat, map, curaddr, sgsize) != 0) { 876 sgsize = MIN(sgsize, 877 PAGE_SIZE - (curaddr & PAGE_MASK)); 878 map->pagesneeded++; 879 } 880 curaddr += sgsize; 881 buflen -= sgsize; 882 } 883 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 884 } 885} 886 887static void 888_bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map, 889 void *buf, bus_size_t buflen, int flags) 890{ 891 vm_offset_t vaddr; 892 vm_offset_t vendaddr; 893 bus_addr_t paddr; 894 895 if (map->pagesneeded == 0) { 896 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 897 " map= %p, pagesneeded= %d", 898 dmat->lowaddr, dmat->boundary, dmat->alignment, 899 map, map->pagesneeded); 900 /* 901 * Count the number of bounce pages 902 * needed in order to complete this transfer 903 */ 904 vaddr = (vm_offset_t)buf; 905 vendaddr = (vm_offset_t)buf + buflen; 906 907 while (vaddr < vendaddr) { 908 if (__predict_true(pmap == kernel_pmap)) 909 paddr = pmap_kextract(vaddr); 910 else 911 paddr = pmap_extract(pmap, vaddr); 912 if (must_bounce(dmat, map, paddr, 913 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 914 PAGE_MASK)))) != 0) { 915 map->pagesneeded++; 916 } 917 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); 918 919 } 920 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 921 } 922} 923 924static int 925_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 926{ 927 928 /* Reserve Necessary Bounce Pages */ 929 mtx_lock(&bounce_lock); 930 if (flags & BUS_DMA_NOWAIT) { 931 if (reserve_bounce_pages(dmat, map, 0) != 0) { 932 map->pagesneeded = 0; 933 mtx_unlock(&bounce_lock); 934 return (ENOMEM); 935 } 936 } else { 937 if (reserve_bounce_pages(dmat, map, 1) != 0) { 938 /* Queue us for resources */ 939 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 940 mtx_unlock(&bounce_lock); 941 return (EINPROGRESS); 942 } 943 } 944 mtx_unlock(&bounce_lock); 945 946 return (0); 947} 948 949/* 950 * Add a single contiguous physical range to the segment list. 951 */ 952static int 953_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 954 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 955{ 956 bus_addr_t baddr, bmask; 957 int seg; 958 959 /* 960 * Make sure we don't cross any boundaries. 961 */ 962 bmask = ~(dmat->boundary - 1); 963 if (dmat->boundary > 0) { 964 baddr = (curaddr + dmat->boundary) & bmask; 965 if (sgsize > (baddr - curaddr)) 966 sgsize = (baddr - curaddr); 967 } 968 969 /* 970 * Insert chunk into a segment, coalescing with 971 * previous segment if possible. 972 */ 973 seg = *segp; 974 if (seg == -1) { 975 seg = 0; 976 segs[seg].ds_addr = curaddr; 977 segs[seg].ds_len = sgsize; 978 } else { 979 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 980 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 981 (dmat->boundary == 0 || 982 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 983 segs[seg].ds_len += sgsize; 984 else { 985 if (++seg >= dmat->nsegments) 986 return (0); 987 segs[seg].ds_addr = curaddr; 988 segs[seg].ds_len = sgsize; 989 } 990 } 991 *segp = seg; 992 return (sgsize); 993} 994 995/* 996 * Utility function to load a physical buffer. segp contains 997 * the starting segment on entrace, and the ending segment on exit. 998 */ 999int 1000_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 1001 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 1002{ 1003 bus_addr_t curaddr; 1004 bus_addr_t sl_end = 0; 1005 bus_size_t sgsize; 1006 struct sync_list *sl; 1007 int error; 1008 1009 if (segs == NULL) 1010 segs = map->segments; 1011 1012 counter_u64_add(maploads_total, 1); 1013 counter_u64_add(maploads_physmem, 1); 1014 1015 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 1016 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 1017 if (map->pagesneeded != 0) { 1018 counter_u64_add(maploads_bounced, 1); 1019 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1020 if (error) 1021 return (error); 1022 } 1023 } 1024 1025 sl = map->slist + map->sync_count - 1; 1026 1027 while (buflen > 0) { 1028 curaddr = buf; 1029 sgsize = MIN(buflen, dmat->maxsegsz); 1030 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, 1031 sgsize)) { 1032 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 1033 curaddr = add_bounce_page(dmat, map, 0, curaddr, 1034 sgsize); 1035 } else { 1036 if (map->sync_count > 0) 1037 sl_end = sl->paddr + sl->datacount; 1038 1039 if (map->sync_count == 0 || curaddr != sl_end) { 1040 if (++map->sync_count > dmat->nsegments) 1041 break; 1042 sl++; 1043 sl->vaddr = 0; 1044 sl->paddr = curaddr; 1045 sl->datacount = sgsize; 1046 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1047 KASSERT(sl->pages != NULL, 1048 ("%s: page at PA:0x%08lx is not in " 1049 "vm_page_array", __func__, curaddr)); 1050 } else 1051 sl->datacount += sgsize; 1052 } 1053 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1054 segp); 1055 if (sgsize == 0) 1056 break; 1057 buf += sgsize; 1058 buflen -= sgsize; 1059 } 1060 1061 /* 1062 * Did we fit? 1063 */ 1064 if (buflen != 0) { 1065 _bus_dmamap_unload(dmat, map); 1066 return (EFBIG); /* XXX better return value here? */ 1067 } 1068 return (0); 1069} 1070 1071int 1072_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1073 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1074 bus_dma_segment_t *segs, int *segp) 1075{ 1076 1077 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1078 segs, segp)); 1079} 1080 1081/* 1082 * Utility function to load a linear buffer. segp contains 1083 * the starting segment on entrance, and the ending segment on exit. 1084 */ 1085int 1086_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1087 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 1088 int *segp) 1089{ 1090 bus_size_t sgsize; 1091 bus_addr_t curaddr; 1092 bus_addr_t sl_pend = 0; 1093 vm_offset_t kvaddr, vaddr, sl_vend = 0; 1094 struct sync_list *sl; 1095 int error; 1096 1097 counter_u64_add(maploads_total, 1); 1098 if (map->flags & DMAMAP_COHERENT) 1099 counter_u64_add(maploads_coherent, 1); 1100 if (map->flags & DMAMAP_DMAMEM_ALLOC) 1101 counter_u64_add(maploads_dmamem, 1); 1102 1103 if (segs == NULL) 1104 segs = map->segments; 1105 1106 if (flags & BUS_DMA_LOAD_MBUF) { 1107 counter_u64_add(maploads_mbuf, 1); 1108 map->flags |= DMAMAP_MBUF; 1109 } 1110 1111 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 1112 _bus_dmamap_count_pages(dmat, pmap, map, buf, buflen, flags); 1113 if (map->pagesneeded != 0) { 1114 counter_u64_add(maploads_bounced, 1); 1115 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1116 if (error) 1117 return (error); 1118 } 1119 } 1120 1121 sl = map->slist + map->sync_count - 1; 1122 vaddr = (vm_offset_t)buf; 1123 1124 while (buflen > 0) { 1125 /* 1126 * Get the physical address for this segment. 1127 */ 1128 if (__predict_true(pmap == kernel_pmap)) { 1129 curaddr = pmap_kextract(vaddr); 1130 kvaddr = vaddr; 1131 } else { 1132 curaddr = pmap_extract(pmap, vaddr); 1133 kvaddr = 0; 1134 } 1135 1136 /* 1137 * Compute the segment size, and adjust counts. 1138 */ 1139 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1140 if (sgsize > dmat->maxsegsz) 1141 sgsize = dmat->maxsegsz; 1142 if (buflen < sgsize) 1143 sgsize = buflen; 1144 1145 if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, 1146 sgsize)) { 1147 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1148 sgsize); 1149 } else { 1150 if (map->sync_count > 0) { 1151 sl_pend = sl->paddr + sl->datacount; 1152 sl_vend = sl->vaddr + sl->datacount; 1153 } 1154 1155 if (map->sync_count == 0 || 1156 (kvaddr != 0 && kvaddr != sl_vend) || 1157 (curaddr != sl_pend)) { 1158 1159 if (++map->sync_count > dmat->nsegments) 1160 goto cleanup; 1161 sl++; 1162 sl->vaddr = kvaddr; 1163 sl->paddr = curaddr; 1164 if (kvaddr != 0) { 1165 sl->pages = NULL; 1166 } else { 1167 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1168 KASSERT(sl->pages != NULL, 1169 ("%s: page at PA:0x%08lx is not " 1170 "in vm_page_array", __func__, 1171 curaddr)); 1172 } 1173 sl->datacount = sgsize; 1174 } else 1175 sl->datacount += sgsize; 1176 } 1177 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1178 segp); 1179 if (sgsize == 0) 1180 break; 1181 vaddr += sgsize; 1182 buflen -= sgsize; 1183 } 1184 1185cleanup: 1186 /* 1187 * Did we fit? 1188 */ 1189 if (buflen != 0) { 1190 _bus_dmamap_unload(dmat, map); 1191 return (EFBIG); /* XXX better return value here? */ 1192 } 1193 return (0); 1194} 1195 1196void 1197__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, 1198 bus_dmamap_callback_t *callback, void *callback_arg) 1199{ 1200 1201 map->mem = *mem; 1202 map->dmat = dmat; 1203 map->callback = callback; 1204 map->callback_arg = callback_arg; 1205} 1206 1207bus_dma_segment_t * 1208_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1209 bus_dma_segment_t *segs, int nsegs, int error) 1210{ 1211 1212 if (segs == NULL) 1213 segs = map->segments; 1214 return (segs); 1215} 1216 1217/* 1218 * Release the mapping held by map. 1219 */ 1220void 1221_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1222{ 1223 struct bounce_page *bpage; 1224 struct bounce_zone *bz; 1225 1226 if ((bz = dmat->bounce_zone) != NULL) { 1227 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1228 STAILQ_REMOVE_HEAD(&map->bpages, links); 1229 free_bounce_page(dmat, bpage); 1230 } 1231 1232 bz = dmat->bounce_zone; 1233 bz->free_bpages += map->pagesreserved; 1234 bz->reserved_bpages -= map->pagesreserved; 1235 map->pagesreserved = 0; 1236 map->pagesneeded = 0; 1237 } 1238 map->sync_count = 0; 1239 map->flags &= ~DMAMAP_MBUF; 1240} 1241 1242static void 1243dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size) 1244{ 1245 /* 1246 * Write back any partial cachelines immediately before and 1247 * after the DMA region. We don't need to round the address 1248 * down to the nearest cacheline or specify the exact size, 1249 * as dcache_wb_poc() will do the rounding for us and works 1250 * at cacheline granularity. 1251 */ 1252 if (va & BUSDMA_DCACHE_MASK) 1253 dcache_wb_poc(va, pa, 1); 1254 if ((va + size) & BUSDMA_DCACHE_MASK) 1255 dcache_wb_poc(va + size, pa + size, 1); 1256 1257 dcache_inv_poc_dma(va, pa, size); 1258} 1259 1260static void 1261dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 1262{ 1263 uint32_t len, offset; 1264 vm_page_t m; 1265 vm_paddr_t pa; 1266 vm_offset_t va, tempva; 1267 bus_size_t size; 1268 1269 offset = sl->paddr & PAGE_MASK; 1270 m = sl->pages; 1271 size = sl->datacount; 1272 pa = sl->paddr; 1273 1274 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 1275 tempva = 0; 1276 if (sl->vaddr == 0) { 1277 len = min(PAGE_SIZE - offset, size); 1278 tempva = pmap_quick_enter_page(m); 1279 va = tempva | offset; 1280 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 1281 ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1282 VM_PAGE_TO_PHYS(m) | offset, pa)); 1283 } else { 1284 len = sl->datacount; 1285 va = sl->vaddr; 1286 } 1287 1288 switch (op) { 1289 case BUS_DMASYNC_PREWRITE: 1290 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1291 dcache_wb_poc(va, pa, len); 1292 break; 1293 case BUS_DMASYNC_PREREAD: 1294 /* 1295 * An mbuf may start in the middle of a cacheline. There 1296 * will be no cpu writes to the beginning of that line 1297 * (which contains the mbuf header) while dma is in 1298 * progress. Handle that case by doing a writeback of 1299 * just the first cacheline before invalidating the 1300 * overall buffer. Any mbuf in a chain may have this 1301 * misalignment. Buffers which are not mbufs bounce if 1302 * they are not aligned to a cacheline. 1303 */ 1304 dma_preread_safe(va, pa, len); 1305 break; 1306 case BUS_DMASYNC_POSTREAD: 1307 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1308 dcache_inv_poc(va, pa, len); 1309 break; 1310 default: 1311 panic("unsupported combination of sync operations: " 1312 "0x%08x\n", op); 1313 } 1314 1315 if (tempva != 0) 1316 pmap_quick_remove_page(tempva); 1317 } 1318} 1319 1320void 1321_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1322{ 1323 struct bounce_page *bpage; 1324 struct sync_list *sl, *end; 1325 vm_offset_t datavaddr, tempvaddr; 1326 1327 if (op == BUS_DMASYNC_POSTWRITE) 1328 return; 1329 1330 /* 1331 * If the buffer was from user space, it is possible that this is not 1332 * the same vm map, especially on a POST operation. It's not clear that 1333 * dma on userland buffers can work at all right now. To be safe, until 1334 * we're able to test direct userland dma, panic on a map mismatch. 1335 */ 1336 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1337 1338 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1339 "performing bounce", __func__, dmat, dmat->flags, op); 1340 1341 /* 1342 * For PREWRITE do a writeback. Clean the caches from the 1343 * innermost to the outermost levels. 1344 */ 1345 if (op & BUS_DMASYNC_PREWRITE) { 1346 while (bpage != NULL) { 1347 tempvaddr = 0; 1348 datavaddr = bpage->datavaddr; 1349 if (datavaddr == 0) { 1350 tempvaddr = pmap_quick_enter_page( 1351 bpage->datapage); 1352 datavaddr = tempvaddr | bpage->dataoffs; 1353 } 1354 bcopy((void *)datavaddr, (void *)bpage->vaddr, 1355 bpage->datacount); 1356 if (tempvaddr != 0) 1357 pmap_quick_remove_page(tempvaddr); 1358 dcache_wb_poc(bpage->vaddr, bpage->busaddr, 1359 bpage->datacount); 1360 bpage = STAILQ_NEXT(bpage, links); 1361 } 1362 dmat->bounce_zone->total_bounced++; 1363 } 1364 1365 /* 1366 * Do an invalidate for PREREAD unless a writeback was already 1367 * done above due to PREWRITE also being set. The reason for a 1368 * PREREAD invalidate is to prevent dirty lines currently in the 1369 * cache from being evicted during the DMA. If a writeback was 1370 * done due to PREWRITE also being set there will be no dirty 1371 * lines and the POSTREAD invalidate handles the rest. The 1372 * invalidate is done from the innermost to outermost level. If 1373 * L2 were done first, a dirty cacheline could be automatically 1374 * evicted from L1 before we invalidated it, re-dirtying the L2. 1375 */ 1376 if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) { 1377 bpage = STAILQ_FIRST(&map->bpages); 1378 while (bpage != NULL) { 1379 dcache_inv_poc_dma(bpage->vaddr, bpage->busaddr, 1380 bpage->datacount); 1381 bpage = STAILQ_NEXT(bpage, links); 1382 } 1383 } 1384 1385 /* 1386 * Re-invalidate the caches on a POSTREAD, even though they were 1387 * already invalidated at PREREAD time. Aggressive prefetching 1388 * due to accesses to other data near the dma buffer could have 1389 * brought buffer data into the caches which is now stale. The 1390 * caches are invalidated from the outermost to innermost; the 1391 * prefetches could be happening right now, and if L1 were 1392 * invalidated first, stale L2 data could be prefetched into L1. 1393 */ 1394 if (op & BUS_DMASYNC_POSTREAD) { 1395 while (bpage != NULL) { 1396 dcache_inv_poc(bpage->vaddr, bpage->busaddr, 1397 bpage->datacount); 1398 tempvaddr = 0; 1399 datavaddr = bpage->datavaddr; 1400 if (datavaddr == 0) { 1401 tempvaddr = pmap_quick_enter_page( 1402 bpage->datapage); 1403 datavaddr = tempvaddr | bpage->dataoffs; 1404 } 1405 bcopy((void *)bpage->vaddr, (void *)datavaddr, 1406 bpage->datacount); 1407 if (tempvaddr != 0) 1408 pmap_quick_remove_page(tempvaddr); 1409 bpage = STAILQ_NEXT(bpage, links); 1410 } 1411 dmat->bounce_zone->total_bounced++; 1412 } 1413 } 1414 1415 /* 1416 * For COHERENT memory no cache maintenance is necessary, but ensure all 1417 * writes have reached memory for the PREWRITE case. No action is 1418 * needed for a PREREAD without PREWRITE also set, because that would 1419 * imply that the cpu had written to the COHERENT buffer and expected 1420 * the dma device to see that change, and by definition a PREWRITE sync 1421 * is required to make that happen. 1422 */ 1423 if (map->flags & DMAMAP_COHERENT) { 1424 if (op & BUS_DMASYNC_PREWRITE) { 1425 dsb(); 1426 cpu_l2cache_drain_writebuf(); 1427 } 1428 return; 1429 } 1430 1431 /* 1432 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All 1433 * the comments about the sequences for flushing cache levels in the 1434 * bounce buffer code above apply here as well. In particular, the fact 1435 * that the sequence is inner-to-outer for PREREAD invalidation and 1436 * outer-to-inner for POSTREAD invalidation is not a mistake. 1437 */ 1438 if (map->sync_count != 0) { 1439 sl = &map->slist[0]; 1440 end = &map->slist[map->sync_count]; 1441 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1442 "performing sync", __func__, dmat, dmat->flags, op); 1443 1444 for ( ; sl != end; ++sl) 1445 dma_dcache_sync(sl, op); 1446 } 1447} 1448 1449static void 1450init_bounce_pages(void *dummy __unused) 1451{ 1452 1453 total_bpages = 0; 1454 STAILQ_INIT(&bounce_zone_list); 1455 STAILQ_INIT(&bounce_map_waitinglist); 1456 STAILQ_INIT(&bounce_map_callbacklist); 1457 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1458} 1459SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1460 1461static struct sysctl_ctx_list * 1462busdma_sysctl_tree(struct bounce_zone *bz) 1463{ 1464 1465 return (&bz->sysctl_tree); 1466} 1467 1468static struct sysctl_oid * 1469busdma_sysctl_tree_top(struct bounce_zone *bz) 1470{ 1471 1472 return (bz->sysctl_tree_top); 1473} 1474 1475static int 1476alloc_bounce_zone(bus_dma_tag_t dmat) 1477{ 1478 struct bounce_zone *bz; 1479 1480 /* Check to see if we already have a suitable zone */ 1481 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1482 if ((dmat->alignment <= bz->alignment) && 1483 (dmat->lowaddr >= bz->lowaddr)) { 1484 dmat->bounce_zone = bz; 1485 return (0); 1486 } 1487 } 1488 1489 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA, 1490 M_NOWAIT | M_ZERO)) == NULL) 1491 return (ENOMEM); 1492 1493 STAILQ_INIT(&bz->bounce_page_list); 1494 bz->free_bpages = 0; 1495 bz->reserved_bpages = 0; 1496 bz->active_bpages = 0; 1497 bz->lowaddr = dmat->lowaddr; 1498 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1499 bz->map_count = 0; 1500 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1501 busdma_zonecount++; 1502 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1503 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1504 dmat->bounce_zone = bz; 1505 1506 sysctl_ctx_init(&bz->sysctl_tree); 1507 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1508 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1509 CTLFLAG_RD, 0, ""); 1510 if (bz->sysctl_tree_top == NULL) { 1511 sysctl_ctx_free(&bz->sysctl_tree); 1512 return (0); /* XXX error code? */ 1513 } 1514 1515 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1516 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1517 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1518 "Total bounce pages"); 1519 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1520 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1521 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1522 "Free bounce pages"); 1523 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1524 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1525 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1526 "Reserved bounce pages"); 1527 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1528 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1529 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1530 "Active bounce pages"); 1531 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1532 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1533 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1534 "Total bounce requests (pages bounced)"); 1535 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1536 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1537 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1538 "Total bounce requests that were deferred"); 1539 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1540 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1541 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1542 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1543 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1544 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1545 1546 return (0); 1547} 1548 1549static int 1550alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1551{ 1552 struct bounce_zone *bz; 1553 int count; 1554 1555 bz = dmat->bounce_zone; 1556 count = 0; 1557 while (numpages > 0) { 1558 struct bounce_page *bpage; 1559 1560 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_BUSDMA, 1561 M_NOWAIT | M_ZERO); 1562 1563 if (bpage == NULL) 1564 break; 1565 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE, 1566 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1567 if (bpage->vaddr == 0) { 1568 free(bpage, M_BUSDMA); 1569 break; 1570 } 1571 bpage->busaddr = pmap_kextract(bpage->vaddr); 1572 mtx_lock(&bounce_lock); 1573 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1574 total_bpages++; 1575 bz->total_bpages++; 1576 bz->free_bpages++; 1577 mtx_unlock(&bounce_lock); 1578 count++; 1579 numpages--; 1580 } 1581 return (count); 1582} 1583 1584static int 1585reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1586{ 1587 struct bounce_zone *bz; 1588 int pages; 1589 1590 mtx_assert(&bounce_lock, MA_OWNED); 1591 bz = dmat->bounce_zone; 1592 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1593 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1594 return (map->pagesneeded - (map->pagesreserved + pages)); 1595 bz->free_bpages -= pages; 1596 bz->reserved_bpages += pages; 1597 map->pagesreserved += pages; 1598 pages = map->pagesneeded - map->pagesreserved; 1599 1600 return (pages); 1601} 1602 1603static bus_addr_t 1604add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1605 bus_addr_t addr, bus_size_t size) 1606{ 1607 struct bounce_zone *bz; 1608 struct bounce_page *bpage; 1609 1610 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1611 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1612 1613 bz = dmat->bounce_zone; 1614 if (map->pagesneeded == 0) 1615 panic("add_bounce_page: map doesn't need any pages"); 1616 map->pagesneeded--; 1617 1618 if (map->pagesreserved == 0) 1619 panic("add_bounce_page: map doesn't need any pages"); 1620 map->pagesreserved--; 1621 1622 mtx_lock(&bounce_lock); 1623 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1624 if (bpage == NULL) 1625 panic("add_bounce_page: free page list is empty"); 1626 1627 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1628 bz->reserved_bpages--; 1629 bz->active_bpages++; 1630 mtx_unlock(&bounce_lock); 1631 1632 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1633 /* Page offset needs to be preserved. */ 1634 bpage->vaddr |= addr & PAGE_MASK; 1635 bpage->busaddr |= addr & PAGE_MASK; 1636 } 1637 bpage->datavaddr = vaddr; 1638 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1639 bpage->dataoffs = addr & PAGE_MASK; 1640 bpage->datacount = size; 1641 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1642 return (bpage->busaddr); 1643} 1644 1645static void 1646free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1647{ 1648 struct bus_dmamap *map; 1649 struct bounce_zone *bz; 1650 1651 bz = dmat->bounce_zone; 1652 bpage->datavaddr = 0; 1653 bpage->datacount = 0; 1654 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1655 /* 1656 * Reset the bounce page to start at offset 0. Other uses 1657 * of this bounce page may need to store a full page of 1658 * data and/or assume it starts on a page boundary. 1659 */ 1660 bpage->vaddr &= ~PAGE_MASK; 1661 bpage->busaddr &= ~PAGE_MASK; 1662 } 1663 1664 mtx_lock(&bounce_lock); 1665 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1666 bz->free_bpages++; 1667 bz->active_bpages--; 1668 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1669 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1670 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1671 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1672 map, links); 1673 busdma_swi_pending = 1; 1674 bz->total_deferred++; 1675 swi_sched(vm_ih, 0); 1676 } 1677 } 1678 mtx_unlock(&bounce_lock); 1679} 1680 1681void 1682busdma_swi(void) 1683{ 1684 bus_dma_tag_t dmat; 1685 struct bus_dmamap *map; 1686 1687 mtx_lock(&bounce_lock); 1688 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1689 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1690 mtx_unlock(&bounce_lock); 1691 dmat = map->dmat; 1692 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1693 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1694 map->callback_arg, BUS_DMA_WAITOK); 1695 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1696 mtx_lock(&bounce_lock); 1697 } 1698 mtx_unlock(&bounce_lock); 1699} 1700