busdma_machdep-v6.c revision 239268
1696Ssundar/*- 2696Ssundar * Copyright (c) 2010 Mark Tinguely 3696Ssundar * Copyright (c) 2004 Olivier Houchard 4877Sattila * Copyright (c) 2002 Peter Grehan 5696Ssundar * Copyright (c) 1997, 1998 Justin T. Gibbs. 6696Ssundar * All rights reserved. 7696Ssundar * 8877Sattila * Redistribution and use in source and binary forms, with or without 9696Ssundar * modification, are permitted provided that the following conditions 10696Ssundar * are met: 11696Ssundar * 1. Redistributions of source code must retain the above copyright 12696Ssundar * notice, this list of conditions, and the following disclaimer, 13696Ssundar * without modification, immediately at the beginning of the file. 14877Sattila * 2. The name of the author may not be used to endorse or promote products 15696Ssundar * derived from this software without specific prior written permission. 16696Ssundar * 17696Ssundar * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18877Sattila * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19696Ssundar * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20696Ssundar * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21696Ssundar * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22696Ssundar * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23696Ssundar * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24696Ssundar * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25696Ssundar * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26696Ssundar * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27696Ssundar * SUCH DAMAGE. 28696Ssundar * 29696Ssundar * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb 30696Ssundar */ 31696Ssundar 32696Ssundar#include <sys/cdefs.h> 33696Ssundar__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep-v6.c 239268 2012-08-15 03:03:03Z gonzo $"); 34 35#define _ARM32_BUS_DMA_PRIVATE 36#include <sys/param.h> 37#include <sys/kdb.h> 38#include <ddb/ddb.h> 39#include <ddb/db_output.h> 40#include <sys/systm.h> 41#include <sys/malloc.h> 42#include <sys/bus.h> 43#include <sys/interrupt.h> 44#include <sys/kernel.h> 45#include <sys/ktr.h> 46#include <sys/lock.h> 47#include <sys/proc.h> 48#include <sys/mutex.h> 49#include <sys/mbuf.h> 50#include <sys/uio.h> 51#include <sys/sysctl.h> 52 53#include <vm/vm.h> 54#include <vm/vm_page.h> 55#include <vm/vm_map.h> 56 57#include <machine/atomic.h> 58#include <machine/bus.h> 59#include <machine/cpufunc.h> 60#include <machine/md_var.h> 61 62#define MAX_BPAGES 64 63#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 64#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 65 66#define FIX_DMAP_BUS_DMASYNC_POSTREAD 67 68struct bounce_zone; 69 70struct bus_dma_tag { 71 bus_dma_tag_t parent; 72 bus_size_t alignment; 73 bus_size_t boundary; 74 bus_addr_t lowaddr; 75 bus_addr_t highaddr; 76 bus_dma_filter_t *filter; 77 void *filterarg; 78 bus_size_t maxsize; 79 u_int nsegments; 80 bus_size_t maxsegsz; 81 int flags; 82 int ref_count; 83 int map_count; 84 bus_dma_lock_t *lockfunc; 85 void *lockfuncarg; 86 bus_dma_segment_t *segments; 87 struct bounce_zone *bounce_zone; 88 /* 89 * DMA range for this tag. If the page doesn't fall within 90 * one of these ranges, an error is returned. The caller 91 * may then decide what to do with the transfer. If the 92 * range pointer is NULL, it is ignored. 93 */ 94 struct arm32_dma_range *ranges; 95 int _nranges; 96 97}; 98 99struct bounce_page { 100 vm_offset_t vaddr; /* kva of bounce buffer */ 101 bus_addr_t busaddr; /* Physical address */ 102 vm_offset_t datavaddr; /* kva of client data */ 103 bus_size_t datacount; /* client data count */ 104 STAILQ_ENTRY(bounce_page) links; 105}; 106 107struct sync_list { 108 vm_offset_t vaddr; /* kva of bounce buffer */ 109 bus_addr_t busaddr; /* Physical address */ 110 bus_size_t datacount; /* client data count */ 111 STAILQ_ENTRY(sync_list) slinks; 112}; 113 114int busdma_swi_pending; 115 116struct bounce_zone { 117 STAILQ_ENTRY(bounce_zone) links; 118 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 119 int total_bpages; 120 int free_bpages; 121 int reserved_bpages; 122 int active_bpages; 123 int total_bounced; 124 int total_deferred; 125 int map_count; 126 bus_size_t alignment; 127 bus_addr_t lowaddr; 128 char zoneid[8]; 129 char lowaddrid[20]; 130 struct sysctl_ctx_list sysctl_tree; 131 struct sysctl_oid *sysctl_tree_top; 132}; 133 134static struct mtx bounce_lock; 135static int total_bpages; 136static int busdma_zonecount; 137static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 138 139SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 140SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 141 "Total bounce pages"); 142 143struct bus_dmamap { 144 struct bp_list bpages; 145 int pagesneeded; 146 int pagesreserved; 147 bus_dma_tag_t dmat; 148 void *buf; /* unmapped buffer pointer */ 149 bus_size_t buflen; /* unmapped buffer length */ 150 pmap_t pmap; 151 bus_dmamap_callback_t *callback; 152 void *callback_arg; 153 STAILQ_ENTRY(bus_dmamap) links; 154 STAILQ_HEAD(,sync_list) slist; 155}; 156 157static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 158static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 159 160static void init_bounce_pages(void *dummy); 161static int alloc_bounce_zone(bus_dma_tag_t dmat); 162static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 163static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 164 int commit); 165static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 166 vm_offset_t vaddr, bus_size_t size); 167static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 168int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 169static int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 170 void *buf, bus_size_t buflen, int flags); 171 172static __inline int 173_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 174{ 175 int i; 176 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 177 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 178 || (lowaddr < phys_avail[i] && 179 highaddr > phys_avail[i])) 180 return (1); 181 } 182 return (0); 183} 184 185static __inline struct arm32_dma_range * 186_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 187 bus_addr_t curaddr) 188{ 189 struct arm32_dma_range *dr; 190 int i; 191 192 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 193 if (curaddr >= dr->dr_sysbase && 194 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 195 return (dr); 196 } 197 198 return (NULL); 199} 200 201/* 202 * Return true if a match is made. 203 * 204 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 205 * 206 * If paddr is within the bounds of the dma tag then call the filter callback 207 * to check for a match, if there is no filter callback then assume a match. 208 */ 209int 210run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 211{ 212 int retval; 213 214 retval = 0; 215 216 do { 217 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 218 || ((paddr & (dmat->alignment - 1)) != 0)) 219 && (dmat->filter == NULL 220 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 221 retval = 1; 222 223 dmat = dmat->parent; 224 } while (retval == 0 && dmat != NULL); 225 return (retval); 226} 227 228/* 229 * Convenience function for manipulating driver locks from busdma (during 230 * busdma_swi, for example). Drivers that don't provide their own locks 231 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 232 * non-mutex locking scheme don't have to use this at all. 233 */ 234void 235busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 236{ 237 struct mtx *dmtx; 238 239 dmtx = (struct mtx *)arg; 240 switch (op) { 241 case BUS_DMA_LOCK: 242 mtx_lock(dmtx); 243 break; 244 case BUS_DMA_UNLOCK: 245 mtx_unlock(dmtx); 246 break; 247 default: 248 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 249 } 250} 251 252/* 253 * dflt_lock should never get called. It gets put into the dma tag when 254 * lockfunc == NULL, which is only valid if the maps that are associated 255 * with the tag are meant to never be defered. 256 * XXX Should have a way to identify which driver is responsible here. 257 */ 258static void 259dflt_lock(void *arg, bus_dma_lock_op_t op) 260{ 261 panic("driver error: busdma dflt_lock called"); 262} 263 264/* 265 * Allocate a device specific dma_tag. 266 */ 267int 268bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 269 bus_size_t boundary, bus_addr_t lowaddr, 270 bus_addr_t highaddr, bus_dma_filter_t *filter, 271 void *filterarg, bus_size_t maxsize, int nsegments, 272 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 273 void *lockfuncarg, bus_dma_tag_t *dmat) 274{ 275 bus_dma_tag_t newtag; 276 int error = 0; 277 278#if 0 279 if (!parent) 280 parent = arm_root_dma_tag; 281#endif 282 283 /* Basic sanity checking */ 284 if (boundary != 0 && boundary < maxsegsz) 285 maxsegsz = boundary; 286 287 /* Return a NULL tag on failure */ 288 *dmat = NULL; 289 290 if (maxsegsz == 0) { 291 return (EINVAL); 292 } 293 294 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 295 M_ZERO | M_NOWAIT); 296 if (newtag == NULL) { 297 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 298 __func__, newtag, 0, error); 299 return (ENOMEM); 300 } 301 302 newtag->parent = parent; 303 newtag->alignment = alignment; 304 newtag->boundary = boundary; 305 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 306 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 307 (PAGE_SIZE - 1); 308 newtag->filter = filter; 309 newtag->filterarg = filterarg; 310 newtag->maxsize = maxsize; 311 newtag->nsegments = nsegments; 312 newtag->maxsegsz = maxsegsz; 313 newtag->flags = flags; 314 newtag->ref_count = 1; /* Count ourself */ 315 newtag->map_count = 0; 316 newtag->ranges = bus_dma_get_range(); 317 newtag->_nranges = bus_dma_get_range_nb(); 318 if (lockfunc != NULL) { 319 newtag->lockfunc = lockfunc; 320 newtag->lockfuncarg = lockfuncarg; 321 } else { 322 newtag->lockfunc = dflt_lock; 323 newtag->lockfuncarg = NULL; 324 } 325 newtag->segments = NULL; 326 327 /* Take into account any restrictions imposed by our parent tag */ 328 if (parent != NULL) { 329 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 330 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 331 if (newtag->boundary == 0) 332 newtag->boundary = parent->boundary; 333 else if (parent->boundary != 0) 334 newtag->boundary = MIN(parent->boundary, 335 newtag->boundary); 336 if ((newtag->filter != NULL) || 337 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 338 newtag->flags |= BUS_DMA_COULD_BOUNCE; 339 if (newtag->filter == NULL) { 340 /* 341 * Short circuit looking at our parent directly 342 * since we have encapsulated all of its information 343 */ 344 newtag->filter = parent->filter; 345 newtag->filterarg = parent->filterarg; 346 newtag->parent = parent->parent; 347 } 348 if (newtag->parent != NULL) 349 atomic_add_int(&parent->ref_count, 1); 350 } 351 352 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 353 || newtag->alignment > 1) 354 newtag->flags |= BUS_DMA_COULD_BOUNCE; 355 356 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 357 (flags & BUS_DMA_ALLOCNOW) != 0) { 358 struct bounce_zone *bz; 359 360 /* Must bounce */ 361 362 if ((error = alloc_bounce_zone(newtag)) != 0) { 363 free(newtag, M_DEVBUF); 364 return (error); 365 } 366 bz = newtag->bounce_zone; 367 368 if (ptoa(bz->total_bpages) < maxsize) { 369 int pages; 370 371 pages = atop(maxsize) - bz->total_bpages; 372 373 /* Add pages to our bounce pool */ 374 if (alloc_bounce_pages(newtag, pages) < pages) 375 error = ENOMEM; 376 } 377 /* Performed initial allocation */ 378 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 379 } else 380 newtag->bounce_zone = NULL; 381 382 if (error != 0) { 383 free(newtag, M_DEVBUF); 384 } else { 385 *dmat = newtag; 386 } 387 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 388 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 389 return (error); 390} 391 392int 393bus_dma_tag_destroy(bus_dma_tag_t dmat) 394{ 395 bus_dma_tag_t dmat_copy; 396 int error; 397 398 error = 0; 399 dmat_copy = dmat; 400 401 if (dmat != NULL) { 402 403 if (dmat->map_count != 0) { 404 error = EBUSY; 405 goto out; 406 } 407 408 while (dmat != NULL) { 409 bus_dma_tag_t parent; 410 411 parent = dmat->parent; 412 atomic_subtract_int(&dmat->ref_count, 1); 413 if (dmat->ref_count == 0) { 414 if (dmat->segments != NULL) 415 free(dmat->segments, M_DEVBUF); 416 free(dmat, M_DEVBUF); 417 /* 418 * Last reference count, so 419 * release our reference 420 * count on our parent. 421 */ 422 dmat = parent; 423 } else 424 dmat = NULL; 425 } 426 } 427out: 428 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 429 return (error); 430} 431 432/* 433 * Allocate a handle for mapping from kva/uva/physical 434 * address space into bus device space. 435 */ 436int 437bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 438{ 439 int error; 440 441 error = 0; 442 443 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 444 M_NOWAIT | M_ZERO); 445 if (*mapp == NULL) { 446 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 447 return (ENOMEM); 448 } 449 STAILQ_INIT(&((*mapp)->slist)); 450 451 if (dmat->segments == NULL) { 452 dmat->segments = (bus_dma_segment_t *)malloc( 453 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 454 M_NOWAIT); 455 if (dmat->segments == NULL) { 456 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 457 __func__, dmat, ENOMEM); 458 free(*mapp, M_DEVBUF); 459 *mapp = NULL; 460 return (ENOMEM); 461 } 462 } 463 /* 464 * Bouncing might be required if the driver asks for an active 465 * exclusion region, a data alignment that is stricter than 1, and/or 466 * an active address boundary. 467 */ 468 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 469 470 /* Must bounce */ 471 struct bounce_zone *bz; 472 int maxpages; 473 474 if (dmat->bounce_zone == NULL) { 475 if ((error = alloc_bounce_zone(dmat)) != 0) { 476 free(*mapp, M_DEVBUF); 477 *mapp = NULL; 478 return (error); 479 } 480 } 481 bz = dmat->bounce_zone; 482 483 /* Initialize the new map */ 484 STAILQ_INIT(&((*mapp)->bpages)); 485 486 /* 487 * Attempt to add pages to our pool on a per-instance 488 * basis up to a sane limit. 489 */ 490 maxpages = MAX_BPAGES; 491 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 492 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 493 int pages; 494 495 pages = MAX(atop(dmat->maxsize), 1); 496 pages = MIN(maxpages - bz->total_bpages, pages); 497 pages = MAX(pages, 1); 498 if (alloc_bounce_pages(dmat, pages) < pages) 499 error = ENOMEM; 500 501 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 502 if (error == 0) 503 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 504 } else { 505 error = 0; 506 } 507 } 508 bz->map_count++; 509 } 510 if (error == 0) 511 dmat->map_count++; 512 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 513 __func__, dmat, dmat->flags, error); 514 return (error); 515} 516 517/* 518 * Destroy a handle for mapping from kva/uva/physical 519 * address space into bus device space. 520 */ 521int 522bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 523{ 524 if (STAILQ_FIRST(&map->bpages) != NULL || 525 STAILQ_FIRST(&map->slist) != NULL) { 526 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 527 __func__, dmat, EBUSY); 528 return (EBUSY); 529 } 530 if (dmat->bounce_zone) 531 dmat->bounce_zone->map_count--; 532 free(map, M_DEVBUF); 533 dmat->map_count--; 534 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 535 return (0); 536} 537 538 539/* 540 * Allocate a piece of memory that can be efficiently mapped into 541 * bus device space based on the constraints lited in the dma tag. 542 * A dmamap to for use with dmamap_load is also allocated. 543 */ 544int 545bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 546 bus_dmamap_t *mapp) 547{ 548 int mflags, len; 549 550 if (flags & BUS_DMA_NOWAIT) 551 mflags = M_NOWAIT; 552 else 553 mflags = M_WAITOK; 554 555 /* ARM non-snooping caches need a map for the VA cache sync structure */ 556 557 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 558 M_NOWAIT | M_ZERO); 559 if (*mapp == NULL) { 560 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 561 __func__, dmat, dmat->flags, ENOMEM); 562 return (ENOMEM); 563 } 564 565 STAILQ_INIT(&((*mapp)->slist)); 566 567 if (dmat->segments == NULL) { 568 dmat->segments = (bus_dma_segment_t *)malloc( 569 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 570 mflags); 571 if (dmat->segments == NULL) { 572 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 573 __func__, dmat, dmat->flags, ENOMEM); 574 free(*mapp, M_DEVBUF); 575 *mapp = NULL; 576 return (ENOMEM); 577 } 578 } 579 580 if (flags & BUS_DMA_ZERO) 581 mflags |= M_ZERO; 582 583 /* 584 * XXX: 585 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact 586 * alignment guarantees of malloc need to be nailed down, and the 587 * code below should be rewritten to take that into account. 588 * 589 * In the meantime, we'll warn the user if malloc gets it wrong. 590 * 591 * allocate at least a cache line. This should help avoid cache 592 * corruption. 593 */ 594 len = max(dmat->maxsize, arm_dcache_align); 595 if (len <= PAGE_SIZE && 596 (dmat->alignment < len) && 597 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 598 *vaddr = malloc(len, M_DEVBUF, mflags); 599 } else { 600 /* 601 * XXX Use Contigmalloc until it is merged into this facility 602 * and handles multi-seg allocations. Nobody is doing 603 * multi-seg allocations yet though. 604 * XXX Certain AGP hardware does. 605 */ 606 *vaddr = contigmalloc(len, M_DEVBUF, mflags, 607 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 608 dmat->boundary); 609 } 610 if (*vaddr == NULL) { 611 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 612 __func__, dmat, dmat->flags, ENOMEM); 613 free(*mapp, M_DEVBUF); 614 *mapp = NULL; 615 return (ENOMEM); 616 } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { 617 printf("bus_dmamem_alloc failed to align memory properly.\n"); 618 } 619 dmat->map_count++; 620 621 if (flags & BUS_DMA_COHERENT) 622 pmap_change_attr((vm_offset_t)*vaddr, len, 623 BUS_DMA_NOCACHE); 624 625 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 626 __func__, dmat, dmat->flags, 0); 627 return (0); 628} 629 630/* 631 * Free a piece of memory and it's allociated dmamap, that was allocated 632 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 633 */ 634void 635bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 636{ 637 int len; 638 639#ifdef mftnotyet 640 pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, ARM_WRITE_BACK); 641#endif 642 len = max(dmat->maxsize, arm_dcache_align); 643 if (len <= PAGE_SIZE && 644 (dmat->alignment < len) && 645 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 646 free(vaddr, M_DEVBUF); 647 else { 648 contigfree(vaddr, len, M_DEVBUF); 649 } 650 dmat->map_count--; 651 free(map, M_DEVBUF); 652 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 653} 654 655static int 656_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 657 void *buf, bus_size_t buflen, int flags) 658{ 659 vm_offset_t vaddr; 660 vm_offset_t vendaddr; 661 bus_addr_t paddr; 662 663 if (map->pagesneeded == 0) { 664 CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" 665 " map= %p, pagesneeded= %d", 666 dmat->lowaddr, dmat->boundary, dmat->alignment, 667 map, map->pagesneeded); 668 /* 669 * Count the number of bounce pages 670 * needed in order to complete this transfer 671 */ 672 vaddr = (vm_offset_t)buf; 673 vendaddr = (vm_offset_t)buf + buflen; 674 675 while (vaddr < vendaddr) { 676 if (__predict_true(map->pmap == pmap_kernel())) 677 paddr = pmap_kextract(vaddr); 678 else 679 paddr = pmap_extract(map->pmap, vaddr); 680 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 681 run_filter(dmat, paddr) != 0) { 682 map->pagesneeded++; 683 } 684 vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); 685 686 } 687 CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); 688 } 689 690 /* Reserve Necessary Bounce Pages */ 691 if (map->pagesneeded != 0) { 692 mtx_lock(&bounce_lock); 693 if (flags & BUS_DMA_NOWAIT) { 694 if (reserve_bounce_pages(dmat, map, 0) != 0) { 695 map->pagesneeded = 0; 696 mtx_unlock(&bounce_lock); 697 return (ENOMEM); 698 } 699 } else { 700 if (reserve_bounce_pages(dmat, map, 1) != 0) { 701 /* Queue us for resources */ 702 map->dmat = dmat; 703 map->buf = buf; 704 map->buflen = buflen; 705 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 706 map, links); 707 mtx_unlock(&bounce_lock); 708 return (EINPROGRESS); 709 } 710 } 711 mtx_unlock(&bounce_lock); 712 } 713 714 return (0); 715} 716 717/* 718 * Utility function to load a linear buffer. lastaddrp holds state 719 * between invocations (for multiple-buffer loads). segp contains 720 * the starting segment on entrace, and the ending segment on exit. 721 * first indicates if this is the first invocation of this function. 722 */ 723static __inline int 724_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 725 bus_dmamap_t map, 726 void *buf, bus_size_t buflen, 727 int flags, 728 bus_addr_t *lastaddrp, 729 bus_dma_segment_t *segs, 730 int *segp, 731 int first) 732{ 733 bus_size_t sgsize; 734 bus_addr_t curaddr, lastaddr, baddr, bmask; 735 vm_offset_t vaddr; 736 struct sync_list *sl; 737 int seg, error; 738 739 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 740 error = _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); 741 if (error) 742 return (error); 743 } 744 745 sl = NULL; 746 vaddr = (vm_offset_t)buf; 747 lastaddr = *lastaddrp; 748 bmask = ~(dmat->boundary - 1); 749 750 for (seg = *segp; buflen > 0 ; ) { 751 /* 752 * Get the physical address for this segment. 753 */ 754 if (__predict_true(map->pmap == pmap_kernel())) 755 curaddr = pmap_kextract(vaddr); 756 else 757 curaddr = pmap_extract(map->pmap, vaddr); 758 759 /* 760 * Compute the segment size, and adjust counts. 761 */ 762 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 763 if (sgsize > dmat->maxsegsz) 764 sgsize = dmat->maxsegsz; 765 if (buflen < sgsize) 766 sgsize = buflen; 767 768 /* 769 * Make sure we don't cross any boundaries. 770 */ 771 if (dmat->boundary > 0) { 772 baddr = (curaddr + dmat->boundary) & bmask; 773 if (sgsize > (baddr - curaddr)) 774 sgsize = (baddr - curaddr); 775 } 776 777 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 778 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 779 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 780 } else { 781 /* add_sync_list(dmat, map, vaddr, sgsize, cflag); */ 782 sl = (struct sync_list *)malloc(sizeof(struct sync_list), 783 M_DEVBUF, M_NOWAIT | M_ZERO); 784 if (sl == NULL) 785 goto cleanup; 786 STAILQ_INSERT_TAIL(&(map->slist), sl, slinks); 787 sl->vaddr = vaddr; 788 sl->datacount = sgsize; 789 sl->busaddr = curaddr; 790 } 791 792 793 if (dmat->ranges) { 794 struct arm32_dma_range *dr; 795 796 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 797 curaddr); 798 if (dr == NULL) { 799 _bus_dmamap_unload(dmat, map); 800 return (EINVAL); 801 } 802 /* 803 * In a valid DMA range. Translate the physical 804 * memory address to an address in the DMA window. 805 */ 806 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 807 } 808 809 /* 810 * Insert chunk into a segment, coalescing with 811 * previous segment if possible. 812 */ 813 if (first) { 814 segs[seg].ds_addr = curaddr; 815 segs[seg].ds_len = sgsize; 816 first = 0; 817 } else { 818 if (curaddr == lastaddr && 819 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 820 (dmat->boundary == 0 || 821 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 822 segs[seg].ds_len += sgsize; 823 else { 824 if (++seg >= dmat->nsegments) 825 break; 826 segs[seg].ds_addr = curaddr; 827 segs[seg].ds_len = sgsize; 828 } 829 } 830 831 lastaddr = curaddr + sgsize; 832 vaddr += sgsize; 833 buflen -= sgsize; 834 } 835 836 *segp = seg; 837 *lastaddrp = lastaddr; 838cleanup: 839 /* 840 * Did we fit? 841 */ 842 if (buflen != 0) { 843 _bus_dmamap_unload(dmat, map); 844 return(EFBIG); /* XXX better return value here? */ 845 } 846 return (0); 847} 848 849/* 850 * Map the buffer buf into bus space using the dmamap map. 851 */ 852int 853bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 854 bus_size_t buflen, bus_dmamap_callback_t *callback, 855 void *callback_arg, int flags) 856{ 857 bus_addr_t lastaddr = 0; 858 int error, nsegs = 0; 859 860 flags |= BUS_DMA_WAITOK; 861 map->callback = callback; 862 map->callback_arg = callback_arg; 863 map->pmap = kernel_pmap; 864 865 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, flags, 866 &lastaddr, dmat->segments, &nsegs, 1); 867 868 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 869 __func__, dmat, dmat->flags, error, nsegs + 1); 870 871 if (error == EINPROGRESS) { 872 return (error); 873 } 874 875 if (error) 876 (*callback)(callback_arg, dmat->segments, 0, error); 877 else 878 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 879 880 /* 881 * Return ENOMEM to the caller so that it can pass it up the stack. 882 * This error only happens when NOWAIT is set, so deferal is disabled. 883 */ 884 if (error == ENOMEM) 885 return (error); 886 887 return (0); 888} 889 890 891/* 892 * Like _bus_dmamap_load(), but for mbufs. 893 */ 894static __inline int 895_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 896 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 897 int flags) 898{ 899 int error; 900 901 M_ASSERTPKTHDR(m0); 902 map->pmap = kernel_pmap; 903 904 flags |= BUS_DMA_NOWAIT; 905 *nsegs = 0; 906 error = 0; 907 if (m0->m_pkthdr.len <= dmat->maxsize) { 908 int first = 1; 909 bus_addr_t lastaddr = 0; 910 struct mbuf *m; 911 912 for (m = m0; m != NULL && error == 0; m = m->m_next) { 913 if (m->m_len > 0) { 914 error = _bus_dmamap_load_buffer(dmat, map, 915 m->m_data, m->m_len, 916 flags, &lastaddr, 917 segs, nsegs, first); 918 first = 0; 919 } 920 } 921 } else { 922 error = EINVAL; 923 } 924 925 /* XXX FIXME: Having to increment nsegs is really annoying */ 926 ++*nsegs; 927 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 928 __func__, dmat, dmat->flags, error, *nsegs); 929 return (error); 930} 931 932int 933bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 934 struct mbuf *m0, 935 bus_dmamap_callback2_t *callback, void *callback_arg, 936 int flags) 937{ 938 int nsegs, error; 939 940 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, 941 flags); 942 943 if (error) { 944 /* force "no valid mappings" in callback */ 945 (*callback)(callback_arg, dmat->segments, 0, 0, error); 946 } else { 947 (*callback)(callback_arg, dmat->segments, 948 nsegs, m0->m_pkthdr.len, error); 949 } 950 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 951 __func__, dmat, dmat->flags, error, nsegs); 952 953 return (error); 954} 955 956int 957bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 958 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 959 int flags) 960{ 961 return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); 962} 963 964/* 965 * Like _bus_dmamap_load(), but for uios. 966 */ 967int 968bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 969 struct uio *uio, 970 bus_dmamap_callback2_t *callback, void *callback_arg, 971 int flags) 972{ 973 bus_addr_t lastaddr; 974 int nsegs, error, first, i; 975 bus_size_t resid; 976 struct iovec *iov; 977 978 flags |= BUS_DMA_NOWAIT; 979 resid = uio->uio_resid; 980 iov = uio->uio_iov; 981 982 if (uio->uio_segflg == UIO_USERSPACE) { 983 KASSERT(uio->uio_td != NULL, 984 ("bus_dmamap_load_uio: USERSPACE but no proc")); 985 map->pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 986 } else 987 map->pmap = kernel_pmap; 988 989 nsegs = 0; 990 error = 0; 991 first = 1; 992 lastaddr = (bus_addr_t) 0; 993 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 994 /* 995 * Now at the first iovec to load. Load each iovec 996 * until we have exhausted the residual count. 997 */ 998 bus_size_t minlen = 999 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1000 caddr_t addr = (caddr_t) iov[i].iov_base; 1001 1002 if (minlen > 0) { 1003 error = _bus_dmamap_load_buffer(dmat, map, 1004 addr, minlen, flags, &lastaddr, 1005 dmat->segments, &nsegs, first); 1006 first = 0; 1007 resid -= minlen; 1008 } 1009 } 1010 1011 if (error) { 1012 /* force "no valid mappings" in callback */ 1013 (*callback)(callback_arg, dmat->segments, 0, 0, error); 1014 } else { 1015 (*callback)(callback_arg, dmat->segments, 1016 nsegs+1, uio->uio_resid, error); 1017 } 1018 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1019 __func__, dmat, dmat->flags, error, nsegs + 1); 1020 return (error); 1021} 1022 1023/* 1024 * Release the mapping held by map. 1025 */ 1026void 1027_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1028{ 1029 struct bounce_page *bpage; 1030 struct bounce_zone *bz; 1031 struct sync_list *sl; 1032 1033 while ((sl = STAILQ_FIRST(&map->slist)) != NULL) { 1034 STAILQ_REMOVE_HEAD(&map->slist, slinks); 1035 free(sl, M_DEVBUF); 1036 } 1037 1038 if ((bz = dmat->bounce_zone) != NULL) { 1039 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1040 STAILQ_REMOVE_HEAD(&map->bpages, links); 1041 free_bounce_page(dmat, bpage); 1042 } 1043 1044 bz = dmat->bounce_zone; 1045 bz->free_bpages += map->pagesreserved; 1046 bz->reserved_bpages -= map->pagesreserved; 1047 map->pagesreserved = 0; 1048 map->pagesneeded = 0; 1049 } 1050} 1051 1052#ifdef notyetbounceuser 1053 /* If busdma uses user pages, then the interrupt handler could 1054 * be use the kernel vm mapping. Both bounce pages and sync list 1055 * do not cross page boundaries. 1056 * Below is a rough sequence that a person would do to fix the 1057 * user page reference in the kernel vmspace. This would be 1058 * done in the dma post routine. 1059 */ 1060void 1061_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len, 1062 pmap_t pmap, int op) 1063{ 1064 bus_size_t sgsize; 1065 bus_addr_t curaddr; 1066 vm_offset_t va; 1067 1068 /* each synclist entry is contained within a single page. 1069 * 1070 * this would be needed if BUS_DMASYNC_POSTxxxx was implemented 1071 */ 1072 curaddr = pmap_extract(pmap, buf); 1073 va = pmap_dma_map(curaddr); 1074 switch (op) { 1075 case SYNC_USER_INV: 1076 cpu_dcache_wb_range(va, sgsize); 1077 break; 1078 1079 case SYNC_USER_COPYTO: 1080 bcopy((void *)va, (void *)bounce, sgsize); 1081 break; 1082 1083 case SYNC_USER_COPYFROM: 1084 bcopy((void *) bounce, (void *)va, sgsize); 1085 break; 1086 1087 default: 1088 break; 1089 } 1090 1091 pmap_dma_unmap(va); 1092} 1093#endif 1094 1095#ifdef ARM_L2_PIPT 1096#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size) 1097#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size) 1098#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size) 1099#else 1100#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size) 1101#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size) 1102#define l2cache_inv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size) 1103#endif 1104 1105void 1106_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1107{ 1108 struct bounce_page *bpage; 1109 struct sync_list *sl; 1110 bus_size_t len, unalign; 1111 vm_offset_t buf, ebuf; 1112#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD 1113 vm_offset_t bbuf; 1114 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1115#endif 1116 int listcount = 0; 1117 1118 /* if buffer was from user space, it it possible that this 1119 * is not the same vm map. The fix is to map each page in 1120 * the buffer into the current address space (KVM) and then 1121 * do the bounce copy or sync list cache operation. 1122 * 1123 * The sync list entries are already broken into 1124 * their respective physical pages. 1125 */ 1126 if (!pmap_dmap_iscurrent(map->pmap)) 1127 printf("_bus_dmamap_sync: wrong user map: %p %x\n", map->pmap, op); 1128 1129 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1130 1131 /* Handle data bouncing. */ 1132 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1133 "performing bounce", __func__, dmat, dmat->flags, op); 1134 1135 if (op & BUS_DMASYNC_PREWRITE) { 1136 while (bpage != NULL) { 1137 bcopy((void *)bpage->datavaddr, 1138 (void *)bpage->vaddr, 1139 bpage->datacount); 1140 cpu_dcache_wb_range((vm_offset_t)bpage->vaddr, 1141 bpage->datacount); 1142 l2cache_wb_range((vm_offset_t)bpage->vaddr, 1143 (vm_offset_t)bpage->busaddr, 1144 bpage->datacount); 1145 bpage = STAILQ_NEXT(bpage, links); 1146 } 1147 dmat->bounce_zone->total_bounced++; 1148 } 1149 1150 if (op & BUS_DMASYNC_POSTREAD) { 1151 if (!pmap_dmap_iscurrent(map->pmap)) 1152 panic("_bus_dmamap_sync: wrong user map. apply fix"); 1153 1154 cpu_dcache_inv_range((vm_offset_t)bpage->vaddr, 1155 bpage->datacount); 1156 l2cache_inv_range((vm_offset_t)bpage->vaddr, 1157 (vm_offset_t)bpage->busaddr, 1158 bpage->datacount); 1159 while (bpage != NULL) { 1160 vm_offset_t startv; 1161 vm_paddr_t startp; 1162 int len; 1163 1164 startv = bpage->vaddr &~ arm_dcache_align_mask; 1165 startp = bpage->busaddr &~ arm_dcache_align_mask; 1166 len = bpage->datacount; 1167 1168 if (startv != bpage->vaddr) 1169 len += bpage->vaddr & arm_dcache_align_mask; 1170 if (len & arm_dcache_align_mask) 1171 len = (len - 1172 (len & arm_dcache_align_mask)) + 1173 arm_dcache_align; 1174 cpu_dcache_inv_range(startv, len); 1175 l2cache_inv_range(startv, startp, len); 1176 bcopy((void *)bpage->vaddr, 1177 (void *)bpage->datavaddr, 1178 bpage->datacount); 1179 bpage = STAILQ_NEXT(bpage, links); 1180 } 1181 dmat->bounce_zone->total_bounced++; 1182 } 1183 } 1184 1185 sl = STAILQ_FIRST(&map->slist); 1186 while (sl) { 1187 listcount++; 1188 sl = STAILQ_NEXT(sl, slinks); 1189 } 1190 if ((sl = STAILQ_FIRST(&map->slist)) != NULL) { 1191 /* ARM caches are not self-snooping for dma */ 1192 1193 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1194 "performing sync", __func__, dmat, dmat->flags, op); 1195 1196 switch (op) { 1197 case BUS_DMASYNC_PREWRITE: 1198 while (sl != NULL) { 1199 cpu_dcache_wb_range(sl->vaddr, sl->datacount); 1200 l2cache_wb_range(sl->vaddr, sl->busaddr, 1201 sl->datacount); 1202 sl = STAILQ_NEXT(sl, slinks); 1203 } 1204 break; 1205 1206 case BUS_DMASYNC_PREREAD: 1207 while (sl != NULL) { 1208 /* write back the unaligned portions */ 1209 vm_paddr_t physaddr = sl->busaddr, ephysaddr; 1210 buf = sl->vaddr; 1211 len = sl->datacount; 1212 ebuf = buf + len; /* end of buffer */ 1213 ephysaddr = physaddr + len; 1214 unalign = buf & arm_dcache_align_mask; 1215 if (unalign) { 1216 /* wbinv leading fragment */ 1217 buf &= ~arm_dcache_align_mask; 1218 physaddr &= ~arm_dcache_align_mask; 1219 cpu_dcache_wbinv_range(buf, 1220 arm_dcache_align); 1221 l2cache_wbinv_range(buf, physaddr, 1222 arm_dcache_align); 1223 buf += arm_dcache_align; 1224 physaddr += arm_dcache_align; 1225 /* number byte in buffer wbinv */ 1226 unalign = arm_dcache_align - unalign; 1227 if (len > unalign) 1228 len -= unalign; 1229 else 1230 len = 0; 1231 } 1232 unalign = ebuf & arm_dcache_align_mask; 1233 if (ebuf > buf && unalign) { 1234 /* wbinv trailing fragment */ 1235 len -= unalign; 1236 ebuf -= unalign; 1237 ephysaddr -= unalign; 1238 cpu_dcache_wbinv_range(ebuf, 1239 arm_dcache_align); 1240 l2cache_wbinv_range(ebuf, ephysaddr, 1241 arm_dcache_align); 1242 } 1243 if (ebuf > buf) { 1244 cpu_dcache_inv_range(buf, len); 1245 l2cache_inv_range(buf, physaddr, len); 1246 } 1247 sl = STAILQ_NEXT(sl, slinks); 1248 } 1249 break; 1250 1251 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1252 while (sl != NULL) { 1253 cpu_dcache_wbinv_range(sl->vaddr, sl->datacount); 1254 l2cache_wbinv_range(sl->vaddr, 1255 sl->busaddr, sl->datacount); 1256 sl = STAILQ_NEXT(sl, slinks); 1257 } 1258 break; 1259 1260#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD 1261 case BUS_DMASYNC_POSTREAD: 1262 if (!pmap_dmap_iscurrent(map->pmap)) 1263 panic("_bus_dmamap_sync: wrong user map. apply fix"); 1264 while (sl != NULL) { 1265 /* write back the unaligned portions */ 1266 vm_paddr_t physaddr; 1267 buf = sl->vaddr; 1268 len = sl->datacount; 1269 physaddr = sl->busaddr; 1270 bbuf = buf & ~arm_dcache_align_mask; 1271 ebuf = buf + len; 1272 physaddr = physaddr & ~arm_dcache_align_mask; 1273 unalign = buf & arm_dcache_align_mask; 1274 if (unalign) { 1275 memcpy(_tmp_cl, (void *)bbuf, unalign); 1276 len += unalign; /* inv entire cache line */ 1277 } 1278 unalign = ebuf & arm_dcache_align_mask; 1279 if (unalign) { 1280 unalign = arm_dcache_align - unalign; 1281 memcpy(_tmp_clend, (void *)ebuf, unalign); 1282 len += unalign; /* inv entire cache line */ 1283 } 1284 /* inv are cache length aligned */ 1285 cpu_dcache_inv_range(bbuf, len); 1286 l2cache_inv_range(bbuf, physaddr, len); 1287 1288 unalign = (vm_offset_t)buf & arm_dcache_align_mask; 1289 if (unalign) { 1290 memcpy((void *)bbuf, _tmp_cl, unalign); 1291 } 1292 unalign = ebuf & arm_dcache_align_mask; 1293 if (unalign) { 1294 unalign = arm_dcache_align - unalign; 1295 memcpy((void *)ebuf, _tmp_clend, unalign); 1296 } 1297 sl = STAILQ_NEXT(sl, slinks); 1298 } 1299 break; 1300#endif /* FIX_DMAP_BUS_DMASYNC_POSTREAD */ 1301 1302 default: 1303 break; 1304 } 1305 } 1306} 1307 1308static void 1309init_bounce_pages(void *dummy __unused) 1310{ 1311 1312 total_bpages = 0; 1313 STAILQ_INIT(&bounce_zone_list); 1314 STAILQ_INIT(&bounce_map_waitinglist); 1315 STAILQ_INIT(&bounce_map_callbacklist); 1316 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1317} 1318SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1319 1320static struct sysctl_ctx_list * 1321busdma_sysctl_tree(struct bounce_zone *bz) 1322{ 1323 return (&bz->sysctl_tree); 1324} 1325 1326static struct sysctl_oid * 1327busdma_sysctl_tree_top(struct bounce_zone *bz) 1328{ 1329 return (bz->sysctl_tree_top); 1330} 1331 1332static int 1333alloc_bounce_zone(bus_dma_tag_t dmat) 1334{ 1335 struct bounce_zone *bz; 1336 1337 /* Check to see if we already have a suitable zone */ 1338 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1339 if ((dmat->alignment <= bz->alignment) 1340 && (dmat->lowaddr >= bz->lowaddr)) { 1341 dmat->bounce_zone = bz; 1342 return (0); 1343 } 1344 } 1345 1346 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1347 M_NOWAIT | M_ZERO)) == NULL) 1348 return (ENOMEM); 1349 1350 STAILQ_INIT(&bz->bounce_page_list); 1351 bz->free_bpages = 0; 1352 bz->reserved_bpages = 0; 1353 bz->active_bpages = 0; 1354 bz->lowaddr = dmat->lowaddr; 1355 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1356 bz->map_count = 0; 1357 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1358 busdma_zonecount++; 1359 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1360 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1361 dmat->bounce_zone = bz; 1362 1363 sysctl_ctx_init(&bz->sysctl_tree); 1364 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1365 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1366 CTLFLAG_RD, 0, ""); 1367 if (bz->sysctl_tree_top == NULL) { 1368 sysctl_ctx_free(&bz->sysctl_tree); 1369 return (0); /* XXX error code? */ 1370 } 1371 1372 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1373 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1374 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1375 "Total bounce pages"); 1376 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1377 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1378 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1379 "Free bounce pages"); 1380 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1381 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1382 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1383 "Reserved bounce pages"); 1384 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1385 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1386 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1387 "Active bounce pages"); 1388 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1389 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1390 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1391 "Total bounce requests"); 1392 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1393 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1394 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1395 "Total bounce requests that were deferred"); 1396 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1397 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1398 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1399 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1400 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1401 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1402 1403 return (0); 1404} 1405 1406static int 1407alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1408{ 1409 struct bounce_zone *bz; 1410 int count; 1411 1412 bz = dmat->bounce_zone; 1413 count = 0; 1414 while (numpages > 0) { 1415 struct bounce_page *bpage; 1416 1417 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1418 M_NOWAIT | M_ZERO); 1419 1420 if (bpage == NULL) 1421 break; 1422 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1423 M_NOWAIT, 0ul, 1424 bz->lowaddr, 1425 PAGE_SIZE, 1426 0); 1427 if (bpage->vaddr == 0) { 1428 free(bpage, M_DEVBUF); 1429 break; 1430 } 1431 bpage->busaddr = pmap_kextract(bpage->vaddr); 1432 mtx_lock(&bounce_lock); 1433 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1434 total_bpages++; 1435 bz->total_bpages++; 1436 bz->free_bpages++; 1437 mtx_unlock(&bounce_lock); 1438 count++; 1439 numpages--; 1440 } 1441 return (count); 1442} 1443 1444static int 1445reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1446{ 1447 struct bounce_zone *bz; 1448 int pages; 1449 1450 mtx_assert(&bounce_lock, MA_OWNED); 1451 bz = dmat->bounce_zone; 1452 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1453 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1454 return (map->pagesneeded - (map->pagesreserved + pages)); 1455 bz->free_bpages -= pages; 1456 bz->reserved_bpages += pages; 1457 map->pagesreserved += pages; 1458 pages = map->pagesneeded - map->pagesreserved; 1459 1460 return (pages); 1461} 1462 1463static bus_addr_t 1464add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1465 bus_size_t size) 1466{ 1467 struct bounce_zone *bz; 1468 struct bounce_page *bpage; 1469 1470 printf("add bounce page\n"); 1471 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1472 KASSERT(map != NULL, 1473 ("add_bounce_page: bad map %p", map)); 1474 1475 bz = dmat->bounce_zone; 1476 if (map->pagesneeded == 0) 1477 panic("add_bounce_page: map doesn't need any pages"); 1478 map->pagesneeded--; 1479 1480 if (map->pagesreserved == 0) 1481 panic("add_bounce_page: map doesn't need any pages"); 1482 map->pagesreserved--; 1483 1484 mtx_lock(&bounce_lock); 1485 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1486 if (bpage == NULL) 1487 panic("add_bounce_page: free page list is empty"); 1488 1489 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1490 bz->reserved_bpages--; 1491 bz->active_bpages++; 1492 mtx_unlock(&bounce_lock); 1493 1494 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1495 /* Page offset needs to be preserved. */ 1496 bpage->vaddr |= vaddr & PAGE_MASK; 1497 bpage->busaddr |= vaddr & PAGE_MASK; 1498 } 1499 bpage->datavaddr = vaddr; 1500 bpage->datacount = size; 1501 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1502 return (bpage->busaddr); 1503} 1504 1505static void 1506free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1507{ 1508 struct bus_dmamap *map; 1509 struct bounce_zone *bz; 1510 1511 bz = dmat->bounce_zone; 1512 bpage->datavaddr = 0; 1513 bpage->datacount = 0; 1514 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1515 /* 1516 * Reset the bounce page to start at offset 0. Other uses 1517 * of this bounce page may need to store a full page of 1518 * data and/or assume it starts on a page boundary. 1519 */ 1520 bpage->vaddr &= ~PAGE_MASK; 1521 bpage->busaddr &= ~PAGE_MASK; 1522 } 1523 1524 mtx_lock(&bounce_lock); 1525 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1526 bz->free_bpages++; 1527 bz->active_bpages--; 1528 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1529 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1530 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1531 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1532 map, links); 1533 busdma_swi_pending = 1; 1534 bz->total_deferred++; 1535 swi_sched(vm_ih, 0); 1536 } 1537 } 1538 mtx_unlock(&bounce_lock); 1539} 1540 1541void 1542busdma_swi(void) 1543{ 1544 bus_dma_tag_t dmat; 1545 struct bus_dmamap *map; 1546 1547 mtx_lock(&bounce_lock); 1548 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1549 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1550 mtx_unlock(&bounce_lock); 1551 dmat = map->dmat; 1552 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1553 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1554 map->callback, map->callback_arg, /*flags*/0); 1555 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1556 mtx_lock(&bounce_lock); 1557 } 1558 mtx_unlock(&bounce_lock); 1559} 1560