1/*- 2 * Copyright (c) 2012 Ian Lepore 3 * Copyright (c) 2004 Olivier Houchard 4 * Copyright (c) 2002 Peter Grehan 5 * Copyright (c) 1997, 1998 Justin T. Gibbs. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification, immediately at the beginning of the file. 14 * 2. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: stable/11/sys/arm/arm/busdma_machdep-v4.c 318976 2017-05-27 07:47:52Z hselasky $"); 34 35/* 36 * ARM bus dma support routines. 37 * 38 * XXX Things to investigate / fix some day... 39 * - What is the earliest that this API can be called? Could there be any 40 * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41 * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42 * bus_dmamap_load() function. This code has historically (and still does) 43 * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44 * error checking because some resource management calls would become WAITOK 45 * and thus "cannot fail." 46 * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47 * creation time, and the result stored in the tag. 48 * - It should be possible to take some shortcuts when mapping a buffer we know 49 * came from the uma(9) allocators based on what we know about such buffers 50 * (aligned, contiguous, etc). 51 * - The allocation of bounce pages could probably be cleaned up, then we could 52 * retire arm_remap_nocache(). 53 */ 54 55#define _ARM32_BUS_DMA_PRIVATE 56#include <sys/param.h> 57#include <sys/systm.h> 58#include <sys/malloc.h> 59#include <sys/bus.h> 60#include <sys/busdma_bufalloc.h> 61#include <sys/counter.h> 62#include <sys/interrupt.h> 63#include <sys/kernel.h> 64#include <sys/ktr.h> 65#include <sys/lock.h> 66#include <sys/memdesc.h> 67#include <sys/proc.h> 68#include <sys/mutex.h> 69#include <sys/sysctl.h> 70#include <sys/uio.h> 71 72#include <vm/vm.h> 73#include <vm/vm_page.h> 74#include <vm/vm_map.h> 75#include <vm/vm_extern.h> 76#include <vm/vm_kern.h> 77 78#include <machine/atomic.h> 79#include <machine/bus.h> 80#include <machine/cpufunc.h> 81#include <machine/md_var.h> 82 83#define MAX_BPAGES 64 84#define MAX_DMA_SEGMENTS 4096 85#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 86#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 87 88struct bounce_zone; 89 90struct bus_dma_tag { 91 bus_dma_tag_t parent; 92 bus_size_t alignment; 93 bus_addr_t boundary; 94 bus_addr_t lowaddr; 95 bus_addr_t highaddr; 96 bus_dma_filter_t *filter; 97 void *filterarg; 98 bus_size_t maxsize; 99 u_int nsegments; 100 bus_size_t maxsegsz; 101 int flags; 102 int ref_count; 103 int map_count; 104 bus_dma_lock_t *lockfunc; 105 void *lockfuncarg; 106 struct bounce_zone *bounce_zone; 107 /* 108 * DMA range for this tag. If the page doesn't fall within 109 * one of these ranges, an error is returned. The caller 110 * may then decide what to do with the transfer. If the 111 * range pointer is NULL, it is ignored. 112 */ 113 struct arm32_dma_range *ranges; 114 int _nranges; 115}; 116 117struct bounce_page { 118 vm_offset_t vaddr; /* kva of bounce buffer */ 119 bus_addr_t busaddr; /* Physical address */ 120 vm_offset_t datavaddr; /* kva of client data */ 121 vm_page_t datapage; /* physical page of client data */ 122 vm_offset_t dataoffs; /* page offset of client data */ 123 bus_size_t datacount; /* client data count */ 124 STAILQ_ENTRY(bounce_page) links; 125}; 126 127struct sync_list { 128 vm_offset_t vaddr; /* kva of client data */ 129 vm_page_t pages; /* starting page of client data */ 130 vm_offset_t dataoffs; /* page offset of client data */ 131 bus_size_t datacount; /* client data count */ 132}; 133 134int busdma_swi_pending; 135 136struct bounce_zone { 137 STAILQ_ENTRY(bounce_zone) links; 138 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 139 int total_bpages; 140 int free_bpages; 141 int reserved_bpages; 142 int active_bpages; 143 int total_bounced; 144 int total_deferred; 145 int map_count; 146 bus_size_t alignment; 147 bus_addr_t lowaddr; 148 char zoneid[8]; 149 char lowaddrid[20]; 150 struct sysctl_ctx_list sysctl_tree; 151 struct sysctl_oid *sysctl_tree_top; 152}; 153 154static struct mtx bounce_lock; 155static int total_bpages; 156static int busdma_zonecount; 157static uint32_t tags_total; 158static uint32_t maps_total; 159static uint32_t maps_dmamem; 160static uint32_t maps_coherent; 161static counter_u64_t maploads_total; 162static counter_u64_t maploads_bounced; 163static counter_u64_t maploads_coherent; 164static counter_u64_t maploads_dmamem; 165static counter_u64_t maploads_mbuf; 166static counter_u64_t maploads_physmem; 167 168static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 169 170SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 171SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0, 172 "Number of active tags"); 173SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0, 174 "Number of active maps"); 175SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0, 176 "Number of active maps for bus_dmamem_alloc buffers"); 177SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0, 178 "Number of active maps with BUS_DMA_COHERENT flag set"); 179SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, 180 &maploads_total, "Number of load operations performed"); 181SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, 182 &maploads_bounced, "Number of load operations that used bounce buffers"); 183SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, 184 &maploads_dmamem, "Number of load operations on BUS_DMA_COHERENT memory"); 185SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, 186 &maploads_dmamem, "Number of load operations on bus_dmamem_alloc buffers"); 187SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, 188 &maploads_mbuf, "Number of load operations for mbufs"); 189SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, 190 &maploads_physmem, "Number of load operations on physical buffers"); 191SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 192 "Total bounce pages"); 193 194struct bus_dmamap { 195 struct bp_list bpages; 196 int pagesneeded; 197 int pagesreserved; 198 bus_dma_tag_t dmat; 199 struct memdesc mem; 200 bus_dmamap_callback_t *callback; 201 void *callback_arg; 202 int flags; 203#define DMAMAP_COHERENT (1 << 0) 204#define DMAMAP_DMAMEM_ALLOC (1 << 1) 205#define DMAMAP_MBUF (1 << 2) 206#define DMAMAP_CACHE_ALIGNED (1 << 3) 207 STAILQ_ENTRY(bus_dmamap) links; 208 bus_dma_segment_t *segments; 209 int sync_count; 210 struct sync_list slist[]; 211}; 212 213static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 214static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 215 216static void init_bounce_pages(void *dummy); 217static int alloc_bounce_zone(bus_dma_tag_t dmat); 218static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 219static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 220 int commit); 221static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 222 vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 223static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 224static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 225 int bufaligned); 226 227/* 228 * ---------------------------------------------------------------------------- 229 * Begin block of code useful to transplant to other implementations. 230 */ 231 232static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 233static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 234 235MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 236MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages"); 237 238static void 239busdma_init(void *dummy) 240{ 241 242 maploads_total = counter_u64_alloc(M_WAITOK); 243 maploads_bounced = counter_u64_alloc(M_WAITOK); 244 maploads_coherent = counter_u64_alloc(M_WAITOK); 245 maploads_dmamem = counter_u64_alloc(M_WAITOK); 246 maploads_mbuf = counter_u64_alloc(M_WAITOK); 247 maploads_physmem = counter_u64_alloc(M_WAITOK); 248 249 /* Create a cache of buffers in standard (cacheable) memory. */ 250 standard_allocator = busdma_bufalloc_create("buffer", 251 arm_dcache_align, /* minimum_alignment */ 252 NULL, /* uma_alloc func */ 253 NULL, /* uma_free func */ 254 0); /* uma_zcreate_flags */ 255 256 /* 257 * Create a cache of buffers in uncacheable memory, to implement the 258 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 259 */ 260 coherent_allocator = busdma_bufalloc_create("coherent", 261 arm_dcache_align, /* minimum_alignment */ 262 busdma_bufalloc_alloc_uncacheable, 263 busdma_bufalloc_free_uncacheable, 264 0); /* uma_zcreate_flags */ 265} 266 267/* 268 * This init historically used SI_SUB_VM, but now the init code requires 269 * malloc(9) using M_BUSDMA memory and the pcpu zones for counter(9), which get 270 * set up by SI_SUB_KMEM and SI_ORDER_LAST, so we'll go right after that by 271 * using SI_SUB_KMEM+1. 272 */ 273SYSINIT(busdma, SI_SUB_KMEM+1, SI_ORDER_FIRST, busdma_init, NULL); 274 275/* 276 * End block of code useful to transplant to other implementations. 277 * ---------------------------------------------------------------------------- 278 */ 279 280/* 281 * Return true if a match is made. 282 * 283 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 284 * 285 * If paddr is within the bounds of the dma tag then call the filter callback 286 * to check for a match, if there is no filter callback then assume a match. 287 */ 288static int 289run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 290{ 291 int retval; 292 293 retval = 0; 294 295 do { 296 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 297 || ((paddr & (dmat->alignment - 1)) != 0)) 298 && (dmat->filter == NULL 299 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 300 retval = 1; 301 302 dmat = dmat->parent; 303 } while (retval == 0 && dmat != NULL); 304 return (retval); 305} 306 307/* 308 * This routine checks the exclusion zone constraints from a tag against the 309 * physical RAM available on the machine. If a tag specifies an exclusion zone 310 * but there's no RAM in that zone, then we avoid allocating resources to bounce 311 * a request, and we can use any memory allocator (as opposed to needing 312 * kmem_alloc_contig() just because it can allocate pages in an address range). 313 * 314 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 315 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 316 * possibly have RAM at an address higher than the highest address we can 317 * express, so we take a fast out. 318 */ 319static __inline int 320_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 321{ 322 int i; 323 324 if (lowaddr >= BUS_SPACE_MAXADDR) 325 return (0); 326 327 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 328 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 329 || (lowaddr < phys_avail[i] && 330 highaddr > phys_avail[i])) 331 return (1); 332 } 333 return (0); 334} 335 336static __inline struct arm32_dma_range * 337_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 338 bus_addr_t curaddr) 339{ 340 struct arm32_dma_range *dr; 341 int i; 342 343 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 344 if (curaddr >= dr->dr_sysbase && 345 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 346 return (dr); 347 } 348 349 return (NULL); 350} 351 352/* 353 * Convenience function for manipulating driver locks from busdma (during 354 * busdma_swi, for example). Drivers that don't provide their own locks 355 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 356 * non-mutex locking scheme don't have to use this at all. 357 */ 358void 359busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 360{ 361 struct mtx *dmtx; 362 363 dmtx = (struct mtx *)arg; 364 switch (op) { 365 case BUS_DMA_LOCK: 366 mtx_lock(dmtx); 367 break; 368 case BUS_DMA_UNLOCK: 369 mtx_unlock(dmtx); 370 break; 371 default: 372 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 373 } 374} 375 376/* 377 * dflt_lock should never get called. It gets put into the dma tag when 378 * lockfunc == NULL, which is only valid if the maps that are associated 379 * with the tag are meant to never be defered. 380 * XXX Should have a way to identify which driver is responsible here. 381 */ 382static void 383dflt_lock(void *arg, bus_dma_lock_op_t op) 384{ 385#ifdef INVARIANTS 386 panic("driver error: busdma dflt_lock called"); 387#else 388 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 389#endif 390} 391 392/* 393 * Allocate a device specific dma_tag. 394 */ 395int 396bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 397 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 398 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 399 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 400 void *lockfuncarg, bus_dma_tag_t *dmat) 401{ 402 bus_dma_tag_t newtag; 403 int error = 0; 404 /* Return a NULL tag on failure */ 405 *dmat = NULL; 406 407 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, M_NOWAIT); 408 if (newtag == NULL) { 409 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 410 __func__, newtag, 0, error); 411 return (ENOMEM); 412 } 413 414 newtag->parent = parent; 415 newtag->alignment = alignment ? alignment : 1; 416 newtag->boundary = boundary; 417 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 418 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 419 newtag->filter = filter; 420 newtag->filterarg = filterarg; 421 newtag->maxsize = maxsize; 422 newtag->nsegments = nsegments; 423 newtag->maxsegsz = maxsegsz; 424 newtag->flags = flags; 425 newtag->ref_count = 1; /* Count ourself */ 426 newtag->map_count = 0; 427 newtag->ranges = bus_dma_get_range(); 428 newtag->_nranges = bus_dma_get_range_nb(); 429 if (lockfunc != NULL) { 430 newtag->lockfunc = lockfunc; 431 newtag->lockfuncarg = lockfuncarg; 432 } else { 433 newtag->lockfunc = dflt_lock; 434 newtag->lockfuncarg = NULL; 435 } 436 437 /* Take into account any restrictions imposed by our parent tag */ 438 if (parent != NULL) { 439 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 440 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 441 if (newtag->boundary == 0) 442 newtag->boundary = parent->boundary; 443 else if (parent->boundary != 0) 444 newtag->boundary = MIN(parent->boundary, 445 newtag->boundary); 446 if ((newtag->filter != NULL) || 447 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 448 newtag->flags |= BUS_DMA_COULD_BOUNCE; 449 if (newtag->filter == NULL) { 450 /* 451 * Short circuit looking at our parent directly 452 * since we have encapsulated all of its information 453 */ 454 newtag->filter = parent->filter; 455 newtag->filterarg = parent->filterarg; 456 newtag->parent = parent->parent; 457 } 458 if (newtag->parent != NULL) 459 atomic_add_int(&parent->ref_count, 1); 460 } 461 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 462 || newtag->alignment > 1) 463 newtag->flags |= BUS_DMA_COULD_BOUNCE; 464 465 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 466 (flags & BUS_DMA_ALLOCNOW) != 0) { 467 struct bounce_zone *bz; 468 469 /* Must bounce */ 470 471 if ((error = alloc_bounce_zone(newtag)) != 0) { 472 free(newtag, M_BUSDMA); 473 return (error); 474 } 475 bz = newtag->bounce_zone; 476 477 if (ptoa(bz->total_bpages) < maxsize) { 478 int pages; 479 480 pages = atop(maxsize) - bz->total_bpages; 481 482 /* Add pages to our bounce pool */ 483 if (alloc_bounce_pages(newtag, pages) < pages) 484 error = ENOMEM; 485 } 486 /* Performed initial allocation */ 487 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 488 } else 489 newtag->bounce_zone = NULL; 490 491 if (error != 0) { 492 free(newtag, M_BUSDMA); 493 } else { 494 atomic_add_32(&tags_total, 1); 495 *dmat = newtag; 496 } 497 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 498 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 499 return (error); 500} 501 502int 503bus_dma_tag_destroy(bus_dma_tag_t dmat) 504{ 505 bus_dma_tag_t dmat_copy; 506 int error; 507 508 error = 0; 509 dmat_copy = dmat; 510 511 if (dmat != NULL) { 512 513 if (dmat->map_count != 0) { 514 error = EBUSY; 515 goto out; 516 } 517 518 while (dmat != NULL) { 519 bus_dma_tag_t parent; 520 521 parent = dmat->parent; 522 atomic_subtract_int(&dmat->ref_count, 1); 523 if (dmat->ref_count == 0) { 524 atomic_subtract_32(&tags_total, 1); 525 free(dmat, M_BUSDMA); 526 /* 527 * Last reference count, so 528 * release our reference 529 * count on our parent. 530 */ 531 dmat = parent; 532 } else 533 dmat = NULL; 534 } 535 } 536out: 537 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 538 return (error); 539} 540 541static int 542allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 543{ 544 int error; 545 546 /* 547 * Bouncing might be required if the driver asks for an active 548 * exclusion region, a data alignment that is stricter than 1, and/or 549 * an active address boundary. 550 */ 551 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 552 553 /* Must bounce */ 554 struct bounce_zone *bz; 555 int maxpages; 556 557 if (dmat->bounce_zone == NULL) { 558 if ((error = alloc_bounce_zone(dmat)) != 0) { 559 return (error); 560 } 561 } 562 bz = dmat->bounce_zone; 563 564 /* Initialize the new map */ 565 STAILQ_INIT(&(map->bpages)); 566 567 /* 568 * Attempt to add pages to our pool on a per-instance 569 * basis up to a sane limit. 570 */ 571 maxpages = MAX_BPAGES; 572 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 573 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 574 int pages; 575 576 pages = MAX(atop(dmat->maxsize), 1); 577 pages = MIN(maxpages - bz->total_bpages, pages); 578 pages = MAX(pages, 1); 579 if (alloc_bounce_pages(dmat, pages) < pages) 580 return (ENOMEM); 581 582 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) 583 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 584 } 585 bz->map_count++; 586 } 587 return (0); 588} 589 590static bus_dmamap_t 591allocate_map(bus_dma_tag_t dmat, int mflags) 592{ 593 int mapsize, segsize; 594 bus_dmamap_t map; 595 596 /* 597 * Allocate the map. The map structure ends with an embedded 598 * variable-sized array of sync_list structures. Following that 599 * we allocate enough extra space to hold the array of bus_dma_segments. 600 */ 601 KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, 602 ("cannot allocate %u dma segments (max is %u)", 603 dmat->nsegments, MAX_DMA_SEGMENTS)); 604 segsize = sizeof(struct bus_dma_segment) * dmat->nsegments; 605 mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments; 606 map = malloc(mapsize + segsize, M_BUSDMA, mflags | M_ZERO); 607 if (map == NULL) { 608 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 609 return (NULL); 610 } 611 map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize); 612 return (map); 613} 614 615/* 616 * Allocate a handle for mapping from kva/uva/physical 617 * address space into bus device space. 618 */ 619int 620bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 621{ 622 bus_dmamap_t map; 623 int error = 0; 624 625 *mapp = map = allocate_map(dmat, M_NOWAIT); 626 if (map == NULL) { 627 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 628 return (ENOMEM); 629 } 630 631 /* 632 * Bouncing might be required if the driver asks for an exclusion 633 * region, a data alignment that is stricter than 1, or DMA that begins 634 * or ends with a partial cacheline. Whether bouncing will actually 635 * happen can't be known until mapping time, but we need to pre-allocate 636 * resources now because we might not be allowed to at mapping time. 637 */ 638 error = allocate_bz_and_pages(dmat, map); 639 if (error != 0) { 640 free(map, M_BUSDMA); 641 *mapp = NULL; 642 return (error); 643 } 644 if (map->flags & DMAMAP_COHERENT) 645 atomic_add_32(&maps_coherent, 1); 646 atomic_add_32(&maps_total, 1); 647 dmat->map_count++; 648 649 return (0); 650} 651 652/* 653 * Destroy a handle for mapping from kva/uva/physical 654 * address space into bus device space. 655 */ 656int 657bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 658{ 659 660 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 661 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 662 __func__, dmat, EBUSY); 663 return (EBUSY); 664 } 665 if (dmat->bounce_zone) 666 dmat->bounce_zone->map_count--; 667 if (map->flags & DMAMAP_COHERENT) 668 atomic_subtract_32(&maps_coherent, 1); 669 atomic_subtract_32(&maps_total, 1); 670 free(map, M_BUSDMA); 671 dmat->map_count--; 672 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 673 return (0); 674} 675 676/* 677 * Allocate a piece of memory that can be efficiently mapped into bus device 678 * space based on the constraints listed in the dma tag. Returns a pointer to 679 * the allocated memory, and a pointer to an associated bus_dmamap. 680 */ 681int 682bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, 683 bus_dmamap_t *mapp) 684{ 685 busdma_bufalloc_t ba; 686 struct busdma_bufzone *bufzone; 687 bus_dmamap_t map; 688 vm_memattr_t memattr; 689 int mflags; 690 691 if (flags & BUS_DMA_NOWAIT) 692 mflags = M_NOWAIT; 693 else 694 mflags = M_WAITOK; 695 if (flags & BUS_DMA_ZERO) 696 mflags |= M_ZERO; 697 698 *mapp = map = allocate_map(dmat, mflags); 699 if (map == NULL) { 700 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 701 __func__, dmat, dmat->flags, ENOMEM); 702 return (ENOMEM); 703 } 704 map->flags = DMAMAP_DMAMEM_ALLOC; 705 706 /* Choose a busdma buffer allocator based on memory type flags. */ 707 if (flags & BUS_DMA_COHERENT) { 708 memattr = VM_MEMATTR_UNCACHEABLE; 709 ba = coherent_allocator; 710 map->flags |= DMAMAP_COHERENT; 711 } else { 712 memattr = VM_MEMATTR_DEFAULT; 713 ba = standard_allocator; 714 } 715 716 /* 717 * Try to find a bufzone in the allocator that holds a cache of buffers 718 * of the right size for this request. If the buffer is too big to be 719 * held in the allocator cache, this returns NULL. 720 */ 721 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 722 723 /* 724 * Allocate the buffer from the uma(9) allocator if... 725 * - It's small enough to be in the allocator (bufzone not NULL). 726 * - The alignment constraint isn't larger than the allocation size 727 * (the allocator aligns buffers to their size boundaries). 728 * - There's no need to handle lowaddr/highaddr exclusion zones. 729 * else allocate non-contiguous pages if... 730 * - The page count that could get allocated doesn't exceed nsegments. 731 * - The alignment constraint isn't larger than a page boundary. 732 * - There are no boundary-crossing constraints. 733 * else allocate a block of contiguous pages because one or more of the 734 * constraints is something that only the contig allocator can fulfill. 735 */ 736 if (bufzone != NULL && dmat->alignment <= bufzone->size && 737 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 738 *vaddr = uma_zalloc(bufzone->umazone, mflags); 739 } else if (dmat->nsegments >= 740 howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) && 741 dmat->alignment <= PAGE_SIZE && 742 (dmat->boundary % PAGE_SIZE) == 0) { 743 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 744 mflags, 0, dmat->lowaddr, memattr); 745 } else { 746 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 747 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 748 memattr); 749 } 750 if (*vaddr == NULL) { 751 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 752 __func__, dmat, dmat->flags, ENOMEM); 753 free(map, M_BUSDMA); 754 *mapp = NULL; 755 return (ENOMEM); 756 } 757 if (map->flags & DMAMAP_COHERENT) 758 atomic_add_32(&maps_coherent, 1); 759 atomic_add_32(&maps_dmamem, 1); 760 atomic_add_32(&maps_total, 1); 761 dmat->map_count++; 762 763 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 764 __func__, dmat, dmat->flags, 0); 765 return (0); 766} 767 768/* 769 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 770 * its associated map. 771 */ 772void 773bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 774{ 775 struct busdma_bufzone *bufzone; 776 busdma_bufalloc_t ba; 777 778 if (map->flags & DMAMAP_COHERENT) 779 ba = coherent_allocator; 780 else 781 ba = standard_allocator; 782 783 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 784 785 if (bufzone != NULL && dmat->alignment <= bufzone->size && 786 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 787 uma_zfree(bufzone->umazone, vaddr); 788 else 789 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 790 791 dmat->map_count--; 792 if (map->flags & DMAMAP_COHERENT) 793 atomic_subtract_32(&maps_coherent, 1); 794 atomic_subtract_32(&maps_total, 1); 795 atomic_subtract_32(&maps_dmamem, 1); 796 free(map, M_BUSDMA); 797 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 798} 799 800static void 801_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 802 bus_size_t buflen, int flags) 803{ 804 bus_addr_t curaddr; 805 bus_size_t sgsize; 806 807 if (map->pagesneeded == 0) { 808 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 809 dmat->lowaddr, dmat->boundary, dmat->alignment); 810 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 811 map, map->pagesneeded); 812 /* 813 * Count the number of bounce pages 814 * needed in order to complete this transfer 815 */ 816 curaddr = buf; 817 while (buflen != 0) { 818 sgsize = MIN(buflen, dmat->maxsegsz); 819 if (run_filter(dmat, curaddr) != 0) { 820 sgsize = MIN(sgsize, 821 PAGE_SIZE - (curaddr & PAGE_MASK)); 822 map->pagesneeded++; 823 } 824 curaddr += sgsize; 825 buflen -= sgsize; 826 } 827 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 828 } 829} 830 831static void 832_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 833 void *buf, bus_size_t buflen, int flags) 834{ 835 vm_offset_t vaddr; 836 vm_offset_t vendaddr; 837 bus_addr_t paddr; 838 839 if (map->pagesneeded == 0) { 840 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 841 dmat->lowaddr, dmat->boundary, dmat->alignment); 842 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 843 map, map->pagesneeded); 844 /* 845 * Count the number of bounce pages 846 * needed in order to complete this transfer 847 */ 848 vaddr = trunc_page((vm_offset_t)buf); 849 vendaddr = (vm_offset_t)buf + buflen; 850 851 while (vaddr < vendaddr) { 852 if (__predict_true(pmap == kernel_pmap)) 853 paddr = pmap_kextract(vaddr); 854 else 855 paddr = pmap_extract(pmap, vaddr); 856 if (run_filter(dmat, paddr) != 0) 857 map->pagesneeded++; 858 vaddr += PAGE_SIZE; 859 } 860 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 861 } 862} 863 864static int 865_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 866{ 867 868 /* Reserve Necessary Bounce Pages */ 869 mtx_lock(&bounce_lock); 870 if (flags & BUS_DMA_NOWAIT) { 871 if (reserve_bounce_pages(dmat, map, 0) != 0) { 872 mtx_unlock(&bounce_lock); 873 return (ENOMEM); 874 } 875 } else { 876 if (reserve_bounce_pages(dmat, map, 1) != 0) { 877 /* Queue us for resources */ 878 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 879 mtx_unlock(&bounce_lock); 880 return (EINPROGRESS); 881 } 882 } 883 mtx_unlock(&bounce_lock); 884 885 return (0); 886} 887 888/* 889 * Add a single contiguous physical range to the segment list. 890 */ 891static int 892_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 893 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 894{ 895 bus_addr_t baddr, bmask; 896 int seg; 897 898 /* 899 * Make sure we don't cross any boundaries. 900 */ 901 bmask = ~(dmat->boundary - 1); 902 if (dmat->boundary > 0) { 903 baddr = (curaddr + dmat->boundary) & bmask; 904 if (sgsize > (baddr - curaddr)) 905 sgsize = (baddr - curaddr); 906 } 907 if (dmat->ranges) { 908 struct arm32_dma_range *dr; 909 910 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 911 curaddr); 912 if (dr == NULL) 913 return (0); 914 /* 915 * In a valid DMA range. Translate the physical 916 * memory address to an address in the DMA window. 917 */ 918 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 919 920 } 921 922 seg = *segp; 923 /* 924 * Insert chunk into a segment, coalescing with 925 * the previous segment if possible. 926 */ 927 if (seg >= 0 && 928 curaddr == segs[seg].ds_addr + segs[seg].ds_len && 929 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 930 (dmat->boundary == 0 || 931 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 932 segs[seg].ds_len += sgsize; 933 } else { 934 if (++seg >= dmat->nsegments) 935 return (0); 936 segs[seg].ds_addr = curaddr; 937 segs[seg].ds_len = sgsize; 938 } 939 *segp = seg; 940 return (sgsize); 941} 942 943/* 944 * Utility function to load a physical buffer. segp contains 945 * the starting segment on entrace, and the ending segment on exit. 946 */ 947int 948_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 949 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 950{ 951 bus_addr_t curaddr; 952 bus_addr_t sl_end = 0; 953 bus_size_t sgsize; 954 struct sync_list *sl; 955 int error; 956 957 if (segs == NULL) 958 segs = map->segments; 959 960 counter_u64_add(maploads_total, 1); 961 counter_u64_add(maploads_physmem, 1); 962 963 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 964 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 965 if (map->pagesneeded != 0) { 966 counter_u64_add(maploads_bounced, 1); 967 error = _bus_dmamap_reserve_pages(dmat, map, flags); 968 if (error) 969 return (error); 970 } 971 } 972 973 sl = map->slist + map->sync_count - 1; 974 975 while (buflen > 0) { 976 curaddr = buf; 977 sgsize = MIN(buflen, dmat->maxsegsz); 978 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 979 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 980 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 981 curaddr = add_bounce_page(dmat, map, 0, curaddr, 982 sgsize); 983 } else { 984 if (map->sync_count > 0) 985 sl_end = VM_PAGE_TO_PHYS(sl->pages) + 986 sl->dataoffs + sl->datacount; 987 988 if (map->sync_count == 0 || curaddr != sl_end) { 989 if (++map->sync_count > dmat->nsegments) 990 break; 991 sl++; 992 sl->vaddr = 0; 993 sl->datacount = sgsize; 994 sl->pages = PHYS_TO_VM_PAGE(curaddr); 995 sl->dataoffs = curaddr & PAGE_MASK; 996 } else 997 sl->datacount += sgsize; 998 } 999 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1000 segp); 1001 if (sgsize == 0) 1002 break; 1003 buf += sgsize; 1004 buflen -= sgsize; 1005 } 1006 1007 /* 1008 * Did we fit? 1009 */ 1010 if (buflen != 0) { 1011 _bus_dmamap_unload(dmat, map); 1012 return (EFBIG); /* XXX better return value here? */ 1013 } 1014 return (0); 1015} 1016 1017int 1018_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1019 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1020 bus_dma_segment_t *segs, int *segp) 1021{ 1022 1023 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1024 segs, segp)); 1025} 1026 1027/* 1028 * Utility function to load a linear buffer. segp contains 1029 * the starting segment on entrance, and the ending segment on exit. 1030 */ 1031int 1032_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1033 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1034 int *segp) 1035{ 1036 bus_size_t sgsize; 1037 bus_addr_t curaddr; 1038 bus_addr_t sl_pend = 0; 1039 struct sync_list *sl; 1040 vm_offset_t kvaddr; 1041 vm_offset_t vaddr = (vm_offset_t)buf; 1042 vm_offset_t sl_vend = 0; 1043 int error = 0; 1044 1045 counter_u64_add(maploads_total, 1); 1046 if (map->flags & DMAMAP_COHERENT) 1047 counter_u64_add(maploads_coherent, 1); 1048 if (map->flags & DMAMAP_DMAMEM_ALLOC) 1049 counter_u64_add(maploads_dmamem, 1); 1050 1051 if (segs == NULL) 1052 segs = map->segments; 1053 if (flags & BUS_DMA_LOAD_MBUF) { 1054 counter_u64_add(maploads_mbuf, 1); 1055 map->flags |= DMAMAP_CACHE_ALIGNED; 1056 } 1057 1058 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1059 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1060 if (map->pagesneeded != 0) { 1061 counter_u64_add(maploads_bounced, 1); 1062 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1063 if (error) 1064 return (error); 1065 } 1066 } 1067 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1068 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1069 1070 sl = map->slist + map->sync_count - 1; 1071 1072 while (buflen > 0) { 1073 /* 1074 * Get the physical address for this segment. 1075 */ 1076 if (__predict_true(pmap == kernel_pmap)) { 1077 curaddr = pmap_kextract(vaddr); 1078 kvaddr = vaddr; 1079 } else { 1080 curaddr = pmap_extract(pmap, vaddr); 1081 map->flags &= ~DMAMAP_COHERENT; 1082 kvaddr = 0; 1083 } 1084 1085 /* 1086 * Compute the segment size, and adjust counts. 1087 */ 1088 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1089 if (sgsize > dmat->maxsegsz) 1090 sgsize = dmat->maxsegsz; 1091 if (buflen < sgsize) 1092 sgsize = buflen; 1093 1094 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1095 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1096 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1097 sgsize); 1098 } else { 1099 if (map->sync_count > 0) { 1100 sl_pend = VM_PAGE_TO_PHYS(sl->pages) + 1101 sl->dataoffs + sl->datacount; 1102 sl_vend = sl->vaddr + sl->datacount; 1103 } 1104 1105 if (map->sync_count == 0 || 1106 (kvaddr != 0 && kvaddr != sl_vend) || 1107 (kvaddr == 0 && curaddr != sl_pend)) { 1108 1109 if (++map->sync_count > dmat->nsegments) 1110 goto cleanup; 1111 sl++; 1112 sl->vaddr = kvaddr; 1113 sl->datacount = sgsize; 1114 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1115 sl->dataoffs = curaddr & PAGE_MASK; 1116 } else 1117 sl->datacount += sgsize; 1118 } 1119 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1120 segp); 1121 if (sgsize == 0) 1122 break; 1123 vaddr += sgsize; 1124 buflen -= sgsize; 1125 } 1126 1127cleanup: 1128 /* 1129 * Did we fit? 1130 */ 1131 if (buflen != 0) { 1132 _bus_dmamap_unload(dmat, map); 1133 return (EFBIG); /* XXX better return value here? */ 1134 } 1135 return (0); 1136} 1137 1138void 1139__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, 1140 bus_dmamap_callback_t *callback, void *callback_arg) 1141{ 1142 1143 KASSERT(dmat != NULL, ("dmatag is NULL")); 1144 KASSERT(map != NULL, ("dmamap is NULL")); 1145 map->mem = *mem; 1146 map->callback = callback; 1147 map->callback_arg = callback_arg; 1148} 1149 1150bus_dma_segment_t * 1151_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1152 bus_dma_segment_t *segs, int nsegs, int error) 1153{ 1154 1155 if (segs == NULL) 1156 segs = map->segments; 1157 return (segs); 1158} 1159 1160/* 1161 * Release the mapping held by map. 1162 */ 1163void 1164_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1165{ 1166 struct bounce_page *bpage; 1167 struct bounce_zone *bz; 1168 1169 if ((bz = dmat->bounce_zone) != NULL) { 1170 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1171 STAILQ_REMOVE_HEAD(&map->bpages, links); 1172 free_bounce_page(dmat, bpage); 1173 } 1174 1175 bz = dmat->bounce_zone; 1176 bz->free_bpages += map->pagesreserved; 1177 bz->reserved_bpages -= map->pagesreserved; 1178 map->pagesreserved = 0; 1179 map->pagesneeded = 0; 1180 } 1181 map->sync_count = 0; 1182 map->flags &= ~DMAMAP_MBUF; 1183} 1184 1185static void 1186bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1187 int bufaligned) 1188{ 1189 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1190 register_t s; 1191 int partial; 1192 1193 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1194 cpu_dcache_wb_range(buf, len); 1195 cpu_l2cache_wb_range(buf, len); 1196 } 1197 1198 /* 1199 * If the caller promises the buffer is properly aligned to a cache line 1200 * (even if the call parms make it look like it isn't) we can avoid 1201 * attempting to preserve the non-DMA part of the cache line in the 1202 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1203 * 1204 * This covers the case of mbufs, where we know how they're aligned and 1205 * know the CPU doesn't touch the header in front of the DMA data area 1206 * during the IO, but it may have touched it right before invoking the 1207 * sync, so a PREREAD writeback is required. 1208 * 1209 * It also handles buffers we created in bus_dmamem_alloc(), which are 1210 * always aligned and padded to cache line size even if the IO length 1211 * isn't a multiple of cache line size. In this case the PREREAD 1212 * writeback probably isn't required, but it's harmless. 1213 */ 1214 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1215 1216 if (op & BUS_DMASYNC_PREREAD) { 1217 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1218 cpu_dcache_inv_range(buf, len); 1219 cpu_l2cache_inv_range(buf, len); 1220 } else { 1221 cpu_dcache_wbinv_range(buf, len); 1222 cpu_l2cache_wbinv_range(buf, len); 1223 } 1224 } 1225 if (op & BUS_DMASYNC_POSTREAD) { 1226 if (partial && !bufaligned) { 1227 s = intr_disable(); 1228 if (buf & arm_dcache_align_mask) 1229 memcpy(_tmp_cl, (void *)(buf & 1230 ~arm_dcache_align_mask), 1231 buf & arm_dcache_align_mask); 1232 if ((buf + len) & arm_dcache_align_mask) 1233 memcpy(_tmp_clend, 1234 (void *)(buf + len), 1235 arm_dcache_align - 1236 ((buf + len) & arm_dcache_align_mask)); 1237 } 1238 cpu_dcache_inv_range(buf, len); 1239 cpu_l2cache_inv_range(buf, len); 1240 if (partial && !bufaligned) { 1241 if (buf & arm_dcache_align_mask) 1242 memcpy((void *)(buf & 1243 ~arm_dcache_align_mask), _tmp_cl, 1244 buf & arm_dcache_align_mask); 1245 if ((buf + len) & arm_dcache_align_mask) 1246 memcpy((void *)(buf + len), 1247 _tmp_clend, arm_dcache_align - 1248 ((buf + len) & arm_dcache_align_mask)); 1249 intr_restore(s); 1250 } 1251 } 1252} 1253 1254static void 1255bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 1256 int bufaligned) 1257{ 1258 vm_offset_t tempvaddr; 1259 vm_page_t curpage; 1260 size_t npages; 1261 1262 if (sl->vaddr != 0) { 1263 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); 1264 return; 1265 } 1266 1267 tempvaddr = 0; 1268 npages = atop(round_page(sl->dataoffs + sl->datacount)); 1269 1270 for (curpage = sl->pages; curpage != sl->pages + npages; ++curpage) { 1271 /* 1272 * If the page is mapped to some other VA that hasn't 1273 * been supplied to busdma, then pmap_quick_enter_page() 1274 * will find all duplicate mappings and mark them 1275 * uncacheable. 1276 * That will also do any necessary wb/inv. Otherwise, 1277 * if the page is truly unmapped, then we don't actually 1278 * need to do cache maintenance. 1279 * XXX: May overwrite DMA'ed data in the POSTREAD 1280 * case where the CPU has written to a cacheline not 1281 * completely covered by the DMA region. 1282 */ 1283 KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) + 1284 ptoa(curpage - sl->pages), 1285 ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1286 VM_PAGE_TO_PHYS(curpage), VM_PAGE_TO_PHYS(sl->pages) + 1287 ptoa(curpage - sl->pages))); 1288 tempvaddr = pmap_quick_enter_page(curpage); 1289 pmap_quick_remove_page(tempvaddr); 1290 } 1291} 1292 1293static void 1294_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1295{ 1296 struct bounce_page *bpage; 1297 vm_offset_t datavaddr, tempvaddr; 1298 1299 if ((op & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) == 0) 1300 return; 1301 1302 STAILQ_FOREACH(bpage, &map->bpages, links) { 1303 tempvaddr = 0; 1304 datavaddr = bpage->datavaddr; 1305 if (op & BUS_DMASYNC_PREWRITE) { 1306 if (datavaddr == 0) { 1307 tempvaddr = 1308 pmap_quick_enter_page(bpage->datapage); 1309 datavaddr = tempvaddr | bpage->dataoffs; 1310 } 1311 bcopy((void *)datavaddr, 1312 (void *)bpage->vaddr, bpage->datacount); 1313 if (tempvaddr != 0) 1314 pmap_quick_remove_page(tempvaddr); 1315 cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1316 cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1317 dmat->bounce_zone->total_bounced++; 1318 } 1319 if (op & BUS_DMASYNC_POSTREAD) { 1320 cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1321 cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1322 if (datavaddr == 0) { 1323 tempvaddr = 1324 pmap_quick_enter_page(bpage->datapage); 1325 datavaddr = tempvaddr | bpage->dataoffs; 1326 } 1327 bcopy((void *)bpage->vaddr, 1328 (void *)datavaddr, bpage->datacount); 1329 if (tempvaddr != 0) 1330 pmap_quick_remove_page(tempvaddr); 1331 dmat->bounce_zone->total_bounced++; 1332 } 1333 } 1334} 1335 1336void 1337_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1338{ 1339 struct sync_list *sl, *end; 1340 int bufaligned; 1341 1342 if (op == BUS_DMASYNC_POSTWRITE) 1343 return; 1344 if (map->flags & DMAMAP_COHERENT) 1345 goto drain; 1346 if (STAILQ_FIRST(&map->bpages)) 1347 _bus_dmamap_sync_bp(dmat, map, op); 1348 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1349 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1350 if (map->sync_count) { 1351 end = &map->slist[map->sync_count]; 1352 for (sl = &map->slist[0]; sl != end; sl++) 1353 bus_dmamap_sync_sl(sl, op, bufaligned); 1354 } 1355 1356drain: 1357 1358 cpu_drain_writebuf(); 1359} 1360 1361static void 1362init_bounce_pages(void *dummy __unused) 1363{ 1364 1365 total_bpages = 0; 1366 STAILQ_INIT(&bounce_zone_list); 1367 STAILQ_INIT(&bounce_map_waitinglist); 1368 STAILQ_INIT(&bounce_map_callbacklist); 1369 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1370} 1371SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1372 1373static struct sysctl_ctx_list * 1374busdma_sysctl_tree(struct bounce_zone *bz) 1375{ 1376 1377 return (&bz->sysctl_tree); 1378} 1379 1380static struct sysctl_oid * 1381busdma_sysctl_tree_top(struct bounce_zone *bz) 1382{ 1383 1384 return (bz->sysctl_tree_top); 1385} 1386 1387static int 1388alloc_bounce_zone(bus_dma_tag_t dmat) 1389{ 1390 struct bounce_zone *bz; 1391 1392 /* Check to see if we already have a suitable zone */ 1393 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1394 if ((dmat->alignment <= bz->alignment) && 1395 (dmat->lowaddr >= bz->lowaddr)) { 1396 dmat->bounce_zone = bz; 1397 return (0); 1398 } 1399 } 1400 1401 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA, 1402 M_NOWAIT | M_ZERO)) == NULL) 1403 return (ENOMEM); 1404 1405 STAILQ_INIT(&bz->bounce_page_list); 1406 bz->free_bpages = 0; 1407 bz->reserved_bpages = 0; 1408 bz->active_bpages = 0; 1409 bz->lowaddr = dmat->lowaddr; 1410 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1411 bz->map_count = 0; 1412 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1413 busdma_zonecount++; 1414 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1415 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1416 dmat->bounce_zone = bz; 1417 1418 sysctl_ctx_init(&bz->sysctl_tree); 1419 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1420 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1421 CTLFLAG_RD, 0, ""); 1422 if (bz->sysctl_tree_top == NULL) { 1423 sysctl_ctx_free(&bz->sysctl_tree); 1424 return (0); /* XXX error code? */ 1425 } 1426 1427 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1428 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1429 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1430 "Total bounce pages"); 1431 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1432 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1433 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1434 "Free bounce pages"); 1435 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1436 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1437 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1438 "Reserved bounce pages"); 1439 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1440 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1441 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1442 "Active bounce pages"); 1443 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1444 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1445 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1446 "Total bounce requests (pages bounced)"); 1447 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1448 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1449 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1450 "Total bounce requests that were deferred"); 1451 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1452 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1453 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1454 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1455 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1456 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1457 1458 return (0); 1459} 1460 1461static int 1462alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1463{ 1464 struct bounce_zone *bz; 1465 int count; 1466 1467 bz = dmat->bounce_zone; 1468 count = 0; 1469 while (numpages > 0) { 1470 struct bounce_page *bpage; 1471 1472 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_BUSDMA, 1473 M_NOWAIT | M_ZERO); 1474 1475 if (bpage == NULL) 1476 break; 1477 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE, 1478 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1479 if (bpage->vaddr == 0) { 1480 free(bpage, M_BUSDMA); 1481 break; 1482 } 1483 bpage->busaddr = pmap_kextract(bpage->vaddr); 1484 mtx_lock(&bounce_lock); 1485 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1486 total_bpages++; 1487 bz->total_bpages++; 1488 bz->free_bpages++; 1489 mtx_unlock(&bounce_lock); 1490 count++; 1491 numpages--; 1492 } 1493 return (count); 1494} 1495 1496static int 1497reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1498{ 1499 struct bounce_zone *bz; 1500 int pages; 1501 1502 mtx_assert(&bounce_lock, MA_OWNED); 1503 bz = dmat->bounce_zone; 1504 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1505 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1506 return (map->pagesneeded - (map->pagesreserved + pages)); 1507 bz->free_bpages -= pages; 1508 bz->reserved_bpages += pages; 1509 map->pagesreserved += pages; 1510 pages = map->pagesneeded - map->pagesreserved; 1511 1512 return (pages); 1513} 1514 1515static bus_addr_t 1516add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1517 bus_addr_t addr, bus_size_t size) 1518{ 1519 struct bounce_zone *bz; 1520 struct bounce_page *bpage; 1521 1522 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1523 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1524 1525 bz = dmat->bounce_zone; 1526 if (map->pagesneeded == 0) 1527 panic("add_bounce_page: map doesn't need any pages"); 1528 map->pagesneeded--; 1529 1530 if (map->pagesreserved == 0) 1531 panic("add_bounce_page: map doesn't need any pages"); 1532 map->pagesreserved--; 1533 1534 mtx_lock(&bounce_lock); 1535 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1536 if (bpage == NULL) 1537 panic("add_bounce_page: free page list is empty"); 1538 1539 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1540 bz->reserved_bpages--; 1541 bz->active_bpages++; 1542 mtx_unlock(&bounce_lock); 1543 1544 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1545 /* Page offset needs to be preserved. */ 1546 bpage->vaddr |= addr & PAGE_MASK; 1547 bpage->busaddr |= addr & PAGE_MASK; 1548 } 1549 bpage->datavaddr = vaddr; 1550 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1551 bpage->dataoffs = addr & PAGE_MASK; 1552 bpage->datacount = size; 1553 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1554 return (bpage->busaddr); 1555} 1556 1557static void 1558free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1559{ 1560 struct bus_dmamap *map; 1561 struct bounce_zone *bz; 1562 1563 bz = dmat->bounce_zone; 1564 bpage->datavaddr = 0; 1565 bpage->datacount = 0; 1566 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1567 /* 1568 * Reset the bounce page to start at offset 0. Other uses 1569 * of this bounce page may need to store a full page of 1570 * data and/or assume it starts on a page boundary. 1571 */ 1572 bpage->vaddr &= ~PAGE_MASK; 1573 bpage->busaddr &= ~PAGE_MASK; 1574 } 1575 1576 mtx_lock(&bounce_lock); 1577 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1578 bz->free_bpages++; 1579 bz->active_bpages--; 1580 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1581 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1582 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1583 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1584 map, links); 1585 busdma_swi_pending = 1; 1586 bz->total_deferred++; 1587 swi_sched(vm_ih, 0); 1588 } 1589 } 1590 mtx_unlock(&bounce_lock); 1591} 1592 1593void 1594busdma_swi(void) 1595{ 1596 bus_dma_tag_t dmat; 1597 struct bus_dmamap *map; 1598 1599 mtx_lock(&bounce_lock); 1600 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1601 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1602 mtx_unlock(&bounce_lock); 1603 dmat = map->dmat; 1604 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1605 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1606 map->callback_arg, BUS_DMA_WAITOK); 1607 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1608 mtx_lock(&bounce_lock); 1609 } 1610 mtx_unlock(&bounce_lock); 1611} 1612