busdma_machdep.c revision 242465
1/*- 2 * Copyright (c) 2006 Oleksandr Tymoshenko 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: head/sys/mips/mips/busdma_machdep.c 242465 2012-11-02 05:22:32Z adrian $"); 31 32/* 33 * MIPS bus dma support routines 34 */ 35 36#include <sys/param.h> 37#include <sys/systm.h> 38#include <sys/malloc.h> 39#include <sys/bus.h> 40#include <sys/interrupt.h> 41#include <sys/lock.h> 42#include <sys/proc.h> 43#include <sys/mutex.h> 44#include <sys/mbuf.h> 45#include <sys/uio.h> 46#include <sys/ktr.h> 47#include <sys/kernel.h> 48#include <sys/sysctl.h> 49 50#include <vm/vm.h> 51#include <vm/vm_page.h> 52#include <vm/vm_map.h> 53 54#include <machine/atomic.h> 55#include <machine/bus.h> 56#include <machine/cache.h> 57#include <machine/cpufunc.h> 58#include <machine/cpuinfo.h> 59#include <machine/md_var.h> 60 61#define MAX_BPAGES 64 62#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 63#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 64 65struct bounce_zone; 66 67struct bus_dma_tag { 68 bus_dma_tag_t parent; 69 bus_size_t alignment; 70 bus_addr_t boundary; 71 bus_addr_t lowaddr; 72 bus_addr_t highaddr; 73 bus_dma_filter_t *filter; 74 void *filterarg; 75 bus_size_t maxsize; 76 u_int nsegments; 77 bus_size_t maxsegsz; 78 int flags; 79 int ref_count; 80 int map_count; 81 bus_dma_lock_t *lockfunc; 82 void *lockfuncarg; 83 bus_dma_segment_t *segments; 84 struct bounce_zone *bounce_zone; 85}; 86 87struct bounce_page { 88 vm_offset_t vaddr; /* kva of bounce buffer */ 89 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */ 90 bus_addr_t busaddr; /* Physical address */ 91 vm_offset_t datavaddr; /* kva of client data */ 92 bus_size_t datacount; /* client data count */ 93 STAILQ_ENTRY(bounce_page) links; 94}; 95 96int busdma_swi_pending; 97 98struct bounce_zone { 99 STAILQ_ENTRY(bounce_zone) links; 100 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 101 int total_bpages; 102 int free_bpages; 103 int reserved_bpages; 104 int active_bpages; 105 int total_bounced; 106 int total_deferred; 107 int map_count; 108 bus_size_t alignment; 109 bus_addr_t lowaddr; 110 char zoneid[8]; 111 char lowaddrid[20]; 112 struct sysctl_ctx_list sysctl_tree; 113 struct sysctl_oid *sysctl_tree_top; 114}; 115 116static struct mtx bounce_lock; 117static int total_bpages; 118static int busdma_zonecount; 119static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 120 121static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 122SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 123 "Total bounce pages"); 124 125#define DMAMAP_LINEAR 0x1 126#define DMAMAP_MBUF 0x2 127#define DMAMAP_UIO 0x4 128#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) 129#define DMAMAP_UNCACHEABLE 0x8 130#define DMAMAP_ALLOCATED 0x10 131#define DMAMAP_MALLOCUSED 0x20 132 133struct bus_dmamap { 134 struct bp_list bpages; 135 int pagesneeded; 136 int pagesreserved; 137 bus_dma_tag_t dmat; 138 int flags; 139 void *buffer; 140 void *origbuffer; 141 void *allocbuffer; 142 TAILQ_ENTRY(bus_dmamap) freelist; 143 int len; 144 STAILQ_ENTRY(bus_dmamap) links; 145 bus_dmamap_callback_t *callback; 146 void *callback_arg; 147 148}; 149 150static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 151static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 152 153static TAILQ_HEAD(,bus_dmamap) dmamap_freelist = 154 TAILQ_HEAD_INITIALIZER(dmamap_freelist); 155 156#define BUSDMA_STATIC_MAPS 500 157static struct bus_dmamap map_pool[BUSDMA_STATIC_MAPS]; 158 159static struct mtx busdma_mtx; 160 161MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 162 163static void init_bounce_pages(void *dummy); 164static int alloc_bounce_zone(bus_dma_tag_t dmat); 165static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 166static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 167 int commit); 168static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 169 vm_offset_t vaddr, bus_size_t size); 170static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 171 172/* Default tag, as most drivers provide no parent tag. */ 173bus_dma_tag_t mips_root_dma_tag; 174 175/* 176 * Return true if a match is made. 177 * 178 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 179 * 180 * If paddr is within the bounds of the dma tag then call the filter callback 181 * to check for a match, if there is no filter callback then assume a match. 182 */ 183static int 184run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 185{ 186 int retval; 187 188 retval = 0; 189 190 do { 191 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 192 || ((paddr & (dmat->alignment - 1)) != 0)) 193 && (dmat->filter == NULL 194 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 195 retval = 1; 196 197 dmat = dmat->parent; 198 } while (retval == 0 && dmat != NULL); 199 return (retval); 200} 201 202static void 203mips_dmamap_freelist_init(void *dummy) 204{ 205 int i; 206 207 for (i = 0; i < BUSDMA_STATIC_MAPS; i++) 208 TAILQ_INSERT_HEAD(&dmamap_freelist, &map_pool[i], freelist); 209} 210 211SYSINIT(busdma, SI_SUB_VM, SI_ORDER_ANY, mips_dmamap_freelist_init, NULL); 212 213/* 214 * Check to see if the specified page is in an allowed DMA range. 215 */ 216 217static __inline int 218bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 219 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 220 int flags, vm_offset_t *lastaddrp, int *segp); 221 222static __inline int 223_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 224{ 225 int i; 226 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 227 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 228 || (lowaddr < phys_avail[i] && 229 highaddr > phys_avail[i])) 230 return (1); 231 } 232 return (0); 233} 234 235/* 236 * Convenience function for manipulating driver locks from busdma (during 237 * busdma_swi, for example). Drivers that don't provide their own locks 238 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 239 * non-mutex locking scheme don't have to use this at all. 240 */ 241void 242busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 243{ 244 struct mtx *dmtx; 245 246 dmtx = (struct mtx *)arg; 247 switch (op) { 248 case BUS_DMA_LOCK: 249 mtx_lock(dmtx); 250 break; 251 case BUS_DMA_UNLOCK: 252 mtx_unlock(dmtx); 253 break; 254 default: 255 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 256 } 257} 258 259/* 260 * dflt_lock should never get called. It gets put into the dma tag when 261 * lockfunc == NULL, which is only valid if the maps that are associated 262 * with the tag are meant to never be defered. 263 * XXX Should have a way to identify which driver is responsible here. 264 */ 265static void 266dflt_lock(void *arg, bus_dma_lock_op_t op) 267{ 268#ifdef INVARIANTS 269 panic("driver error: busdma dflt_lock called"); 270#else 271 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 272#endif 273} 274 275static __inline bus_dmamap_t 276_busdma_alloc_dmamap(void) 277{ 278 bus_dmamap_t map; 279 280 mtx_lock(&busdma_mtx); 281 map = TAILQ_FIRST(&dmamap_freelist); 282 if (map) 283 TAILQ_REMOVE(&dmamap_freelist, map, freelist); 284 mtx_unlock(&busdma_mtx); 285 if (!map) { 286 map = malloc(sizeof(*map), M_DEVBUF, M_NOWAIT | M_ZERO); 287 if (map) 288 map->flags = DMAMAP_ALLOCATED; 289 } else 290 map->flags = 0; 291 STAILQ_INIT(&map->bpages); 292 return (map); 293} 294 295static __inline void 296_busdma_free_dmamap(bus_dmamap_t map) 297{ 298 if (map->flags & DMAMAP_ALLOCATED) 299 free(map, M_DEVBUF); 300 else { 301 mtx_lock(&busdma_mtx); 302 TAILQ_INSERT_HEAD(&dmamap_freelist, map, freelist); 303 mtx_unlock(&busdma_mtx); 304 } 305} 306 307/* 308 * Allocate a device specific dma_tag. 309 */ 310#define SEG_NB 1024 311 312int 313bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 314 bus_addr_t boundary, bus_addr_t lowaddr, 315 bus_addr_t highaddr, bus_dma_filter_t *filter, 316 void *filterarg, bus_size_t maxsize, int nsegments, 317 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 318 void *lockfuncarg, bus_dma_tag_t *dmat) 319{ 320 bus_dma_tag_t newtag; 321 int error = 0; 322 /* Return a NULL tag on failure */ 323 *dmat = NULL; 324 if (!parent) 325 parent = mips_root_dma_tag; 326 327 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 328 if (newtag == NULL) { 329 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 330 __func__, newtag, 0, error); 331 return (ENOMEM); 332 } 333 334 newtag->parent = parent; 335 newtag->alignment = alignment; 336 newtag->boundary = boundary; 337 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 338 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 339 newtag->filter = filter; 340 newtag->filterarg = filterarg; 341 newtag->maxsize = maxsize; 342 newtag->nsegments = nsegments; 343 newtag->maxsegsz = maxsegsz; 344 newtag->flags = flags; 345 if (cpuinfo.cache_coherent_dma) 346 newtag->flags |= BUS_DMA_COHERENT; 347 newtag->ref_count = 1; /* Count ourself */ 348 newtag->map_count = 0; 349 if (lockfunc != NULL) { 350 newtag->lockfunc = lockfunc; 351 newtag->lockfuncarg = lockfuncarg; 352 } else { 353 newtag->lockfunc = dflt_lock; 354 newtag->lockfuncarg = NULL; 355 } 356 newtag->segments = NULL; 357 358 /* 359 * Take into account any restrictions imposed by our parent tag 360 */ 361 if (parent != NULL) { 362 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 363 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 364 if (newtag->boundary == 0) 365 newtag->boundary = parent->boundary; 366 else if (parent->boundary != 0) 367 newtag->boundary = 368 MIN(parent->boundary, newtag->boundary); 369 if ((newtag->filter != NULL) || 370 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 371 newtag->flags |= BUS_DMA_COULD_BOUNCE; 372 if (newtag->filter == NULL) { 373 /* 374 * Short circuit looking at our parent directly 375 * since we have encapsulated all of its information 376 */ 377 newtag->filter = parent->filter; 378 newtag->filterarg = parent->filterarg; 379 newtag->parent = parent->parent; 380 } 381 if (newtag->parent != NULL) 382 atomic_add_int(&parent->ref_count, 1); 383 } 384 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 385 || newtag->alignment > 1) 386 newtag->flags |= BUS_DMA_COULD_BOUNCE; 387 388 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 389 (flags & BUS_DMA_ALLOCNOW) != 0) { 390 struct bounce_zone *bz; 391 392 /* Must bounce */ 393 394 if ((error = alloc_bounce_zone(newtag)) != 0) { 395 free(newtag, M_DEVBUF); 396 return (error); 397 } 398 bz = newtag->bounce_zone; 399 400 if (ptoa(bz->total_bpages) < maxsize) { 401 int pages; 402 403 pages = atop(maxsize) - bz->total_bpages; 404 405 /* Add pages to our bounce pool */ 406 if (alloc_bounce_pages(newtag, pages) < pages) 407 error = ENOMEM; 408 } 409 /* Performed initial allocation */ 410 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 411 } else 412 newtag->bounce_zone = NULL; 413 if (error != 0) 414 free(newtag, M_DEVBUF); 415 else 416 *dmat = newtag; 417 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 418 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 419 420 return (error); 421} 422 423int 424bus_dma_tag_destroy(bus_dma_tag_t dmat) 425{ 426#ifdef KTR 427 bus_dma_tag_t dmat_copy = dmat; 428#endif 429 430 if (dmat != NULL) { 431 if (dmat->map_count != 0) 432 return (EBUSY); 433 434 while (dmat != NULL) { 435 bus_dma_tag_t parent; 436 437 parent = dmat->parent; 438 atomic_subtract_int(&dmat->ref_count, 1); 439 if (dmat->ref_count == 0) { 440 if (dmat->segments != NULL) 441 free(dmat->segments, M_DEVBUF); 442 free(dmat, M_DEVBUF); 443 /* 444 * Last reference count, so 445 * release our reference 446 * count on our parent. 447 */ 448 dmat = parent; 449 } else 450 dmat = NULL; 451 } 452 } 453 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 454 455 return (0); 456} 457 458#include <sys/kdb.h> 459/* 460 * Allocate a handle for mapping from kva/uva/physical 461 * address space into bus device space. 462 */ 463int 464bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 465{ 466 bus_dmamap_t newmap; 467 int error = 0; 468 469 if (dmat->segments == NULL) { 470 dmat->segments = (bus_dma_segment_t *)malloc( 471 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 472 M_NOWAIT); 473 if (dmat->segments == NULL) { 474 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 475 __func__, dmat, ENOMEM); 476 return (ENOMEM); 477 } 478 } 479 480 newmap = _busdma_alloc_dmamap(); 481 if (newmap == NULL) { 482 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 483 return (ENOMEM); 484 } 485 *mapp = newmap; 486 newmap->dmat = dmat; 487 newmap->allocbuffer = NULL; 488 dmat->map_count++; 489 490 /* 491 * Bouncing might be required if the driver asks for an active 492 * exclusion region, a data alignment that is stricter than 1, and/or 493 * an active address boundary. 494 */ 495 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 496 497 /* Must bounce */ 498 struct bounce_zone *bz; 499 int maxpages; 500 501 if (dmat->bounce_zone == NULL) { 502 if ((error = alloc_bounce_zone(dmat)) != 0) { 503 _busdma_free_dmamap(newmap); 504 *mapp = NULL; 505 return (error); 506 } 507 } 508 bz = dmat->bounce_zone; 509 510 /* Initialize the new map */ 511 STAILQ_INIT(&((*mapp)->bpages)); 512 513 /* 514 * Attempt to add pages to our pool on a per-instance 515 * basis up to a sane limit. 516 */ 517 maxpages = MAX_BPAGES; 518 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 519 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 520 int pages; 521 522 pages = MAX(atop(dmat->maxsize), 1); 523 pages = MIN(maxpages - bz->total_bpages, pages); 524 pages = MAX(pages, 1); 525 if (alloc_bounce_pages(dmat, pages) < pages) 526 error = ENOMEM; 527 528 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 529 if (error == 0) 530 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 531 } else { 532 error = 0; 533 } 534 } 535 bz->map_count++; 536 } 537 538 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 539 __func__, dmat, dmat->flags, error); 540 541 return (0); 542} 543 544/* 545 * Destroy a handle for mapping from kva/uva/physical 546 * address space into bus device space. 547 */ 548int 549bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 550{ 551 552 if (STAILQ_FIRST(&map->bpages) != NULL) { 553 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 554 __func__, dmat, EBUSY); 555 return (EBUSY); 556 } 557 if (dmat->bounce_zone) 558 dmat->bounce_zone->map_count--; 559 dmat->map_count--; 560 _busdma_free_dmamap(map); 561 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 562 return (0); 563} 564 565/* 566 * Allocate a piece of memory that can be efficiently mapped into 567 * bus device space based on the constraints lited in the dma tag. 568 * A dmamap to for use with dmamap_load is also allocated. 569 */ 570int 571bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 572 bus_dmamap_t *mapp) 573{ 574 bus_dmamap_t newmap = NULL; 575 576 int mflags; 577 578 if (flags & BUS_DMA_NOWAIT) 579 mflags = M_NOWAIT; 580 else 581 mflags = M_WAITOK; 582 if (dmat->segments == NULL) { 583 dmat->segments = (bus_dma_segment_t *)malloc( 584 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 585 mflags); 586 if (dmat->segments == NULL) { 587 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 588 __func__, dmat, dmat->flags, ENOMEM); 589 return (ENOMEM); 590 } 591 } 592 if (flags & BUS_DMA_ZERO) 593 mflags |= M_ZERO; 594 595 newmap = _busdma_alloc_dmamap(); 596 if (newmap == NULL) { 597 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 598 __func__, dmat, dmat->flags, ENOMEM); 599 return (ENOMEM); 600 } 601 dmat->map_count++; 602 *mapp = newmap; 603 newmap->dmat = dmat; 604 605 /* 606 * If all the memory is coherent with DMA then we don't need to 607 * do anything special for a coherent mapping request. 608 */ 609 if (dmat->flags & BUS_DMA_COHERENT) 610 flags &= ~BUS_DMA_COHERENT; 611 612 /* 613 * Allocate uncacheable memory if all else fails. 614 */ 615 if (flags & BUS_DMA_COHERENT) 616 newmap->flags |= DMAMAP_UNCACHEABLE; 617 618 if (dmat->maxsize <= PAGE_SIZE && 619 (dmat->alignment < dmat->maxsize) && 620 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr) && 621 !(newmap->flags & DMAMAP_UNCACHEABLE)) { 622 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 623 newmap->flags |= DMAMAP_MALLOCUSED; 624 } else { 625 /* 626 * XXX Use Contigmalloc until it is merged into this facility 627 * and handles multi-seg allocations. Nobody is doing 628 * multi-seg allocations yet though. 629 */ 630 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 631 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 632 dmat->boundary); 633 } 634 if (*vaddr == NULL) { 635 if (newmap != NULL) { 636 _busdma_free_dmamap(newmap); 637 dmat->map_count--; 638 } 639 *mapp = NULL; 640 return (ENOMEM); 641 } 642 643 if (newmap->flags & DMAMAP_UNCACHEABLE) { 644 void *tmpaddr = (void *)*vaddr; 645 646 if (tmpaddr) { 647 tmpaddr = (void *)pmap_mapdev(vtophys(tmpaddr), 648 dmat->maxsize); 649 newmap->origbuffer = *vaddr; 650 newmap->allocbuffer = tmpaddr; 651 mips_dcache_wbinv_range((vm_offset_t)*vaddr, 652 dmat->maxsize); 653 *vaddr = tmpaddr; 654 } else 655 newmap->origbuffer = newmap->allocbuffer = NULL; 656 } else 657 newmap->origbuffer = newmap->allocbuffer = NULL; 658 659 return (0); 660} 661 662/* 663 * Free a piece of memory and it's allocated dmamap, that was allocated 664 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 665 */ 666void 667bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 668{ 669 if (map->allocbuffer) { 670 KASSERT(map->allocbuffer == vaddr, 671 ("Trying to freeing the wrong DMA buffer")); 672 vaddr = map->origbuffer; 673 } 674 675 if (map->flags & DMAMAP_UNCACHEABLE) 676 pmap_unmapdev((vm_offset_t)map->allocbuffer, dmat->maxsize); 677 if (map->flags & DMAMAP_MALLOCUSED) 678 free(vaddr, M_DEVBUF); 679 else 680 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 681 682 dmat->map_count--; 683 _busdma_free_dmamap(map); 684 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 685} 686 687static int 688_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 689 void *buf, bus_size_t buflen, int flags) 690{ 691 vm_offset_t vaddr; 692 vm_offset_t vendaddr; 693 bus_addr_t paddr; 694 695 if ((map->pagesneeded == 0)) { 696 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 697 dmat->lowaddr, dmat->boundary, dmat->alignment); 698 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 699 map, map->pagesneeded); 700 /* 701 * Count the number of bounce pages 702 * needed in order to complete this transfer 703 */ 704 vaddr = (vm_offset_t)buf; 705 vendaddr = (vm_offset_t)buf + buflen; 706 707 while (vaddr < vendaddr) { 708 bus_size_t sg_len; 709 710 KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap")); 711 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 712 paddr = pmap_kextract(vaddr); 713 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 714 run_filter(dmat, paddr) != 0) { 715 sg_len = roundup2(sg_len, dmat->alignment); 716 map->pagesneeded++; 717 } 718 vaddr += sg_len; 719 } 720 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 721 } 722 723 /* Reserve Necessary Bounce Pages */ 724 if (map->pagesneeded != 0) { 725 mtx_lock(&bounce_lock); 726 if (flags & BUS_DMA_NOWAIT) { 727 if (reserve_bounce_pages(dmat, map, 0) != 0) { 728 mtx_unlock(&bounce_lock); 729 return (ENOMEM); 730 } 731 } else { 732 if (reserve_bounce_pages(dmat, map, 1) != 0) { 733 /* Queue us for resources */ 734 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 735 map, links); 736 mtx_unlock(&bounce_lock); 737 return (EINPROGRESS); 738 } 739 } 740 mtx_unlock(&bounce_lock); 741 } 742 743 return (0); 744} 745 746/* 747 * Utility function to load a linear buffer. lastaddrp holds state 748 * between invocations (for multiple-buffer loads). segp contains 749 * the starting segment on entrance, and the ending segment on exit. 750 * first indicates if this is the first invocation of this function. 751 */ 752static __inline int 753bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 754 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 755 int flags, vm_offset_t *lastaddrp, int *segp) 756{ 757 bus_size_t sgsize; 758 bus_addr_t curaddr, lastaddr, baddr, bmask; 759 vm_offset_t vaddr = (vm_offset_t)buf; 760 int seg; 761 int error = 0; 762 763 lastaddr = *lastaddrp; 764 bmask = ~(dmat->boundary - 1); 765 766 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 767 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, 768 flags); 769 if (error) 770 return (error); 771 } 772 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 773 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 774 775 for (seg = *segp; buflen > 0 ; ) { 776 /* 777 * Get the physical address for this segment. 778 * 779 * XXX Don't support checking for coherent mappings 780 * XXX in user address space. 781 */ 782 KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap")); 783 curaddr = pmap_kextract(vaddr); 784 785 /* 786 * Compute the segment size, and adjust counts. 787 */ 788 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 789 if (sgsize > dmat->maxsegsz) 790 sgsize = dmat->maxsegsz; 791 if (buflen < sgsize) 792 sgsize = buflen; 793 794 /* 795 * Make sure we don't cross any boundaries. 796 */ 797 if (dmat->boundary > 0) { 798 baddr = (curaddr + dmat->boundary) & bmask; 799 if (sgsize > (baddr - curaddr)) 800 sgsize = (baddr - curaddr); 801 } 802 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 803 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 804 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 805 } 806 807 /* 808 * Insert chunk into a segment, coalescing with 809 * the previous segment if possible. 810 */ 811 if (seg >= 0 && curaddr == lastaddr && 812 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 813 (dmat->boundary == 0 || 814 (segs[seg].ds_addr & bmask) == 815 (curaddr & bmask))) { 816 segs[seg].ds_len += sgsize; 817 goto segdone; 818 } else { 819 if (++seg >= dmat->nsegments) 820 break; 821 segs[seg].ds_addr = curaddr; 822 segs[seg].ds_len = sgsize; 823 } 824 if (error) 825 break; 826segdone: 827 lastaddr = curaddr + sgsize; 828 vaddr += sgsize; 829 buflen -= sgsize; 830 } 831 832 *segp = seg; 833 *lastaddrp = lastaddr; 834 835 /* 836 * Did we fit? 837 */ 838 if (buflen != 0) 839 error = EFBIG; /* XXX better return value here? */ 840 return (error); 841} 842 843/* 844 * Map the buffer buf into bus space using the dmamap map. 845 */ 846int 847bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 848 bus_size_t buflen, bus_dmamap_callback_t *callback, 849 void *callback_arg, int flags) 850{ 851 vm_offset_t lastaddr = 0; 852 int error, nsegs = -1; 853 854 KASSERT(dmat != NULL, ("dmatag is NULL")); 855 KASSERT(map != NULL, ("dmamap is NULL")); 856 map->callback = callback; 857 map->callback_arg = callback_arg; 858 map->flags &= ~DMAMAP_TYPE_MASK; 859 map->flags |= DMAMAP_LINEAR; 860 map->buffer = buf; 861 map->len = buflen; 862 error = bus_dmamap_load_buffer(dmat, 863 dmat->segments, map, buf, buflen, kernel_pmap, 864 flags, &lastaddr, &nsegs); 865 if (error == EINPROGRESS) 866 return (error); 867 if (error) 868 (*callback)(callback_arg, NULL, 0, error); 869 else 870 (*callback)(callback_arg, dmat->segments, nsegs + 1, error); 871 872 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 873 __func__, dmat, dmat->flags, nsegs + 1, error); 874 875 return (error); 876} 877 878/* 879 * Like bus_dmamap_load(), but for mbufs. 880 */ 881int 882bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 883 bus_dmamap_callback2_t *callback, void *callback_arg, 884 int flags) 885{ 886 int nsegs = -1, error = 0; 887 888 M_ASSERTPKTHDR(m0); 889 890 map->flags &= ~DMAMAP_TYPE_MASK; 891 map->flags |= DMAMAP_MBUF; 892 map->buffer = m0; 893 map->len = 0; 894 if (m0->m_pkthdr.len <= dmat->maxsize) { 895 vm_offset_t lastaddr = 0; 896 struct mbuf *m; 897 898 for (m = m0; m != NULL && error == 0; m = m->m_next) { 899 if (m->m_len > 0) { 900 error = bus_dmamap_load_buffer(dmat, 901 dmat->segments, map, m->m_data, m->m_len, 902 kernel_pmap, flags, &lastaddr, &nsegs); 903 map->len += m->m_len; 904 } 905 } 906 } else { 907 error = EINVAL; 908 } 909 910 if (error) { 911 /* 912 * force "no valid mappings" on error in callback. 913 */ 914 (*callback)(callback_arg, dmat->segments, 0, 0, error); 915 } else { 916 (*callback)(callback_arg, dmat->segments, nsegs + 1, 917 m0->m_pkthdr.len, error); 918 } 919 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 920 __func__, dmat, dmat->flags, error, nsegs + 1); 921 922 return (error); 923} 924 925int 926bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 927 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 928 int flags) 929{ 930 int error = 0; 931 M_ASSERTPKTHDR(m0); 932 933 flags |= BUS_DMA_NOWAIT; 934 *nsegs = -1; 935 map->flags &= ~DMAMAP_TYPE_MASK; 936 map->flags |= DMAMAP_MBUF; 937 map->buffer = m0; 938 map->len = 0; 939 if (m0->m_pkthdr.len <= dmat->maxsize) { 940 vm_offset_t lastaddr = 0; 941 struct mbuf *m; 942 943 for (m = m0; m != NULL && error == 0; m = m->m_next) { 944 if (m->m_len > 0) { 945 error = bus_dmamap_load_buffer(dmat, segs, map, 946 m->m_data, m->m_len, 947 kernel_pmap, flags, &lastaddr, 948 nsegs); 949 map->len += m->m_len; 950 } 951 } 952 } else { 953 error = EINVAL; 954 } 955 956 /* XXX FIXME: Having to increment nsegs is really annoying */ 957 ++*nsegs; 958 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 959 __func__, dmat, dmat->flags, error, *nsegs); 960 return (error); 961} 962 963/* 964 * Like bus_dmamap_load(), but for uios. 965 */ 966int 967bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 968 bus_dmamap_callback2_t *callback, void *callback_arg, 969 int flags) 970{ 971 vm_offset_t lastaddr = 0; 972 int nsegs, i, error; 973 bus_size_t resid; 974 struct iovec *iov; 975 struct pmap *pmap; 976 977 resid = uio->uio_resid; 978 iov = uio->uio_iov; 979 map->flags &= ~DMAMAP_TYPE_MASK; 980 map->flags |= DMAMAP_UIO; 981 map->buffer = uio; 982 map->len = 0; 983 984 if (uio->uio_segflg == UIO_USERSPACE) { 985 KASSERT(uio->uio_td != NULL, 986 ("bus_dmamap_load_uio: USERSPACE but no proc")); 987 /* XXX: pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); */ 988 panic("can't do it yet"); 989 } else 990 pmap = kernel_pmap; 991 992 error = 0; 993 nsegs = -1; 994 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 995 /* 996 * Now at the first iovec to load. Load each iovec 997 * until we have exhausted the residual count. 998 */ 999 bus_size_t minlen = 1000 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1001 caddr_t addr = (caddr_t) iov[i].iov_base; 1002 1003 if (minlen > 0) { 1004 error = bus_dmamap_load_buffer(dmat, dmat->segments, 1005 map, addr, minlen, pmap, flags, &lastaddr, &nsegs); 1006 1007 map->len += minlen; 1008 resid -= minlen; 1009 } 1010 } 1011 1012 if (error) { 1013 /* 1014 * force "no valid mappings" on error in callback. 1015 */ 1016 (*callback)(callback_arg, dmat->segments, 0, 0, error); 1017 } else { 1018 (*callback)(callback_arg, dmat->segments, nsegs+1, 1019 uio->uio_resid, error); 1020 } 1021 1022 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1023 __func__, dmat, dmat->flags, error, nsegs + 1); 1024 return (error); 1025} 1026 1027/* 1028 * Release the mapping held by map. 1029 */ 1030void 1031_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1032{ 1033 struct bounce_page *bpage; 1034 1035 map->flags &= ~DMAMAP_TYPE_MASK; 1036 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1037 STAILQ_REMOVE_HEAD(&map->bpages, links); 1038 free_bounce_page(dmat, bpage); 1039 } 1040 return; 1041} 1042 1043static void 1044bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op) 1045{ 1046 char tmp_cl[mips_pdcache_linesize], tmp_clend[mips_pdcache_linesize]; 1047 vm_offset_t buf_cl, buf_clend; 1048 vm_size_t size_cl, size_clend; 1049 int cache_linesize_mask = mips_pdcache_linesize - 1; 1050 1051 /* 1052 * dcache invalidation operates on cache line aligned addresses 1053 * and could modify areas of memory that share the same cache line 1054 * at the beginning and the ending of the buffer. In order to 1055 * prevent a data loss we save these chunks in temporary buffer 1056 * before invalidation and restore them afer it 1057 */ 1058 buf_cl = (vm_offset_t)buf & ~cache_linesize_mask; 1059 size_cl = (vm_offset_t)buf & cache_linesize_mask; 1060 buf_clend = (vm_offset_t)buf + len; 1061 size_clend = (mips_pdcache_linesize - 1062 (buf_clend & cache_linesize_mask)) & cache_linesize_mask; 1063 1064 switch (op) { 1065 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1066 case BUS_DMASYNC_POSTREAD: 1067 1068 /* 1069 * Save buffers that might be modified by invalidation 1070 */ 1071 if (size_cl) 1072 memcpy (tmp_cl, (void*)buf_cl, size_cl); 1073 if (size_clend) 1074 memcpy (tmp_clend, (void*)buf_clend, size_clend); 1075 mips_dcache_inv_range((vm_offset_t)buf, len); 1076 /* 1077 * Restore them 1078 */ 1079 if (size_cl) 1080 memcpy ((void*)buf_cl, tmp_cl, size_cl); 1081 if (size_clend) 1082 memcpy ((void*)buf_clend, tmp_clend, size_clend); 1083 /* 1084 * Copies above have brought corresponding memory 1085 * cache lines back into dirty state. Write them back 1086 * out and invalidate affected cache lines again if 1087 * necessary. 1088 */ 1089 if (size_cl) 1090 mips_dcache_wbinv_range((vm_offset_t)buf_cl, size_cl); 1091 if (size_clend && (size_cl == 0 || 1092 buf_clend - buf_cl > mips_pdcache_linesize)) 1093 mips_dcache_wbinv_range((vm_offset_t)buf_clend, 1094 size_clend); 1095 break; 1096 1097 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE: 1098 mips_dcache_wbinv_range((vm_offset_t)buf_cl, len); 1099 break; 1100 1101 case BUS_DMASYNC_PREREAD: 1102 /* 1103 * Save buffers that might be modified by invalidation 1104 */ 1105 if (size_cl) 1106 memcpy (tmp_cl, (void *)buf_cl, size_cl); 1107 if (size_clend) 1108 memcpy (tmp_clend, (void *)buf_clend, size_clend); 1109 mips_dcache_inv_range((vm_offset_t)buf, len); 1110 /* 1111 * Restore them 1112 */ 1113 if (size_cl) 1114 memcpy ((void *)buf_cl, tmp_cl, size_cl); 1115 if (size_clend) 1116 memcpy ((void *)buf_clend, tmp_clend, size_clend); 1117 /* 1118 * Copies above have brought corresponding memory 1119 * cache lines back into dirty state. Write them back 1120 * out and invalidate affected cache lines again if 1121 * necessary. 1122 */ 1123 if (size_cl) 1124 mips_dcache_wbinv_range((vm_offset_t)buf_cl, size_cl); 1125 if (size_clend && (size_cl == 0 || 1126 buf_clend - buf_cl > mips_pdcache_linesize)) 1127 mips_dcache_wbinv_range((vm_offset_t)buf_clend, 1128 size_clend); 1129 break; 1130 1131 case BUS_DMASYNC_PREWRITE: 1132 mips_dcache_wb_range((vm_offset_t)buf, len); 1133 break; 1134 } 1135} 1136 1137static void 1138_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1139{ 1140 struct bounce_page *bpage; 1141 1142 STAILQ_FOREACH(bpage, &map->bpages, links) { 1143 if (op & BUS_DMASYNC_PREWRITE) { 1144 bcopy((void *)bpage->datavaddr, 1145 (void *)(bpage->vaddr_nocache != 0 ? 1146 bpage->vaddr_nocache : bpage->vaddr), 1147 bpage->datacount); 1148 if (bpage->vaddr_nocache == 0) { 1149 mips_dcache_wb_range(bpage->vaddr, 1150 bpage->datacount); 1151 } 1152 dmat->bounce_zone->total_bounced++; 1153 } 1154 if (op & BUS_DMASYNC_POSTREAD) { 1155 if (bpage->vaddr_nocache == 0) { 1156 mips_dcache_inv_range(bpage->vaddr, 1157 bpage->datacount); 1158 } 1159 bcopy((void *)(bpage->vaddr_nocache != 0 ? 1160 bpage->vaddr_nocache : bpage->vaddr), 1161 (void *)bpage->datavaddr, bpage->datacount); 1162 dmat->bounce_zone->total_bounced++; 1163 } 1164 } 1165} 1166 1167static __inline int 1168_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len) 1169{ 1170 struct bounce_page *bpage; 1171 1172 STAILQ_FOREACH(bpage, &map->bpages, links) { 1173 if ((vm_offset_t)buf >= bpage->datavaddr && 1174 (vm_offset_t)buf + len <= bpage->datavaddr + 1175 bpage->datacount) 1176 return (1); 1177 } 1178 return (0); 1179 1180} 1181 1182void 1183_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1184{ 1185 struct mbuf *m; 1186 struct uio *uio; 1187 int resid; 1188 struct iovec *iov; 1189 1190 if (op == BUS_DMASYNC_POSTWRITE) 1191 return; 1192 if (STAILQ_FIRST(&map->bpages)) 1193 _bus_dmamap_sync_bp(dmat, map, op); 1194 1195 if (dmat->flags & BUS_DMA_COHERENT) 1196 return; 1197 1198 if (map->flags & DMAMAP_UNCACHEABLE) 1199 return; 1200 1201 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1202 switch(map->flags & DMAMAP_TYPE_MASK) { 1203 case DMAMAP_LINEAR: 1204 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) 1205 bus_dmamap_sync_buf(map->buffer, map->len, op); 1206 break; 1207 case DMAMAP_MBUF: 1208 m = map->buffer; 1209 while (m) { 1210 if (m->m_len > 0 && 1211 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) 1212 bus_dmamap_sync_buf(m->m_data, m->m_len, op); 1213 m = m->m_next; 1214 } 1215 break; 1216 case DMAMAP_UIO: 1217 uio = map->buffer; 1218 iov = uio->uio_iov; 1219 resid = uio->uio_resid; 1220 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1221 bus_size_t minlen = resid < iov[i].iov_len ? resid : 1222 iov[i].iov_len; 1223 if (minlen > 0) { 1224 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, 1225 minlen)) 1226 bus_dmamap_sync_buf(iov[i].iov_base, 1227 minlen, op); 1228 resid -= minlen; 1229 } 1230 } 1231 break; 1232 default: 1233 break; 1234 } 1235} 1236 1237static void 1238init_bounce_pages(void *dummy __unused) 1239{ 1240 1241 total_bpages = 0; 1242 STAILQ_INIT(&bounce_zone_list); 1243 STAILQ_INIT(&bounce_map_waitinglist); 1244 STAILQ_INIT(&bounce_map_callbacklist); 1245 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1246} 1247SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1248 1249static struct sysctl_ctx_list * 1250busdma_sysctl_tree(struct bounce_zone *bz) 1251{ 1252 return (&bz->sysctl_tree); 1253} 1254 1255static struct sysctl_oid * 1256busdma_sysctl_tree_top(struct bounce_zone *bz) 1257{ 1258 return (bz->sysctl_tree_top); 1259} 1260 1261static int 1262alloc_bounce_zone(bus_dma_tag_t dmat) 1263{ 1264 struct bounce_zone *bz; 1265 1266 /* Check to see if we already have a suitable zone */ 1267 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1268 if ((dmat->alignment <= bz->alignment) 1269 && (dmat->lowaddr >= bz->lowaddr)) { 1270 dmat->bounce_zone = bz; 1271 return (0); 1272 } 1273 } 1274 1275 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1276 M_NOWAIT | M_ZERO)) == NULL) 1277 return (ENOMEM); 1278 1279 STAILQ_INIT(&bz->bounce_page_list); 1280 bz->free_bpages = 0; 1281 bz->reserved_bpages = 0; 1282 bz->active_bpages = 0; 1283 bz->lowaddr = dmat->lowaddr; 1284 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1285 bz->map_count = 0; 1286 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1287 busdma_zonecount++; 1288 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1289 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1290 dmat->bounce_zone = bz; 1291 1292 sysctl_ctx_init(&bz->sysctl_tree); 1293 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1294 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1295 CTLFLAG_RD, 0, ""); 1296 if (bz->sysctl_tree_top == NULL) { 1297 sysctl_ctx_free(&bz->sysctl_tree); 1298 return (0); /* XXX error code? */ 1299 } 1300 1301 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1302 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1303 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1304 "Total bounce pages"); 1305 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1306 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1307 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1308 "Free bounce pages"); 1309 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1310 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1311 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1312 "Reserved bounce pages"); 1313 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1314 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1315 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1316 "Active bounce pages"); 1317 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1318 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1319 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1320 "Total bounce requests"); 1321 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1322 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1323 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1324 "Total bounce requests that were deferred"); 1325 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1326 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1327 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1328 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1329 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1330 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1331 1332 return (0); 1333} 1334 1335static int 1336alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1337{ 1338 struct bounce_zone *bz; 1339 int count; 1340 1341 bz = dmat->bounce_zone; 1342 count = 0; 1343 while (numpages > 0) { 1344 struct bounce_page *bpage; 1345 1346 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1347 M_NOWAIT | M_ZERO); 1348 1349 if (bpage == NULL) 1350 break; 1351 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1352 M_NOWAIT, 0ul, 1353 bz->lowaddr, 1354 PAGE_SIZE, 1355 0); 1356 if (bpage->vaddr == 0) { 1357 free(bpage, M_DEVBUF); 1358 break; 1359 } 1360 bpage->busaddr = pmap_kextract(bpage->vaddr); 1361 bpage->vaddr_nocache = 1362 (vm_offset_t)pmap_mapdev(bpage->busaddr, PAGE_SIZE); 1363 mtx_lock(&bounce_lock); 1364 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1365 total_bpages++; 1366 bz->total_bpages++; 1367 bz->free_bpages++; 1368 mtx_unlock(&bounce_lock); 1369 count++; 1370 numpages--; 1371 } 1372 return (count); 1373} 1374 1375static int 1376reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1377{ 1378 struct bounce_zone *bz; 1379 int pages; 1380 1381 mtx_assert(&bounce_lock, MA_OWNED); 1382 bz = dmat->bounce_zone; 1383 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1384 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1385 return (map->pagesneeded - (map->pagesreserved + pages)); 1386 bz->free_bpages -= pages; 1387 bz->reserved_bpages += pages; 1388 map->pagesreserved += pages; 1389 pages = map->pagesneeded - map->pagesreserved; 1390 1391 return (pages); 1392} 1393 1394static bus_addr_t 1395add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1396 bus_size_t size) 1397{ 1398 struct bounce_zone *bz; 1399 struct bounce_page *bpage; 1400 1401 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1402 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1403 1404 bz = dmat->bounce_zone; 1405 if (map->pagesneeded == 0) 1406 panic("add_bounce_page: map doesn't need any pages"); 1407 map->pagesneeded--; 1408 1409 if (map->pagesreserved == 0) 1410 panic("add_bounce_page: map doesn't need any pages"); 1411 map->pagesreserved--; 1412 1413 mtx_lock(&bounce_lock); 1414 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1415 if (bpage == NULL) 1416 panic("add_bounce_page: free page list is empty"); 1417 1418 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1419 bz->reserved_bpages--; 1420 bz->active_bpages++; 1421 mtx_unlock(&bounce_lock); 1422 1423 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1424 /* Page offset needs to be preserved. */ 1425 bpage->vaddr |= vaddr & PAGE_MASK; 1426 bpage->busaddr |= vaddr & PAGE_MASK; 1427 } 1428 bpage->datavaddr = vaddr; 1429 bpage->datacount = size; 1430 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1431 return (bpage->busaddr); 1432} 1433 1434static void 1435free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1436{ 1437 struct bus_dmamap *map; 1438 struct bounce_zone *bz; 1439 1440 bz = dmat->bounce_zone; 1441 bpage->datavaddr = 0; 1442 bpage->datacount = 0; 1443 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1444 /* 1445 * Reset the bounce page to start at offset 0. Other uses 1446 * of this bounce page may need to store a full page of 1447 * data and/or assume it starts on a page boundary. 1448 */ 1449 bpage->vaddr &= ~PAGE_MASK; 1450 bpage->busaddr &= ~PAGE_MASK; 1451 } 1452 1453 mtx_lock(&bounce_lock); 1454 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1455 bz->free_bpages++; 1456 bz->active_bpages--; 1457 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1458 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1459 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1460 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1461 map, links); 1462 busdma_swi_pending = 1; 1463 bz->total_deferred++; 1464 swi_sched(vm_ih, 0); 1465 } 1466 } 1467 mtx_unlock(&bounce_lock); 1468} 1469 1470void 1471busdma_swi(void) 1472{ 1473 bus_dma_tag_t dmat; 1474 struct bus_dmamap *map; 1475 1476 mtx_lock(&bounce_lock); 1477 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1478 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1479 mtx_unlock(&bounce_lock); 1480 dmat = map->dmat; 1481 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1482 bus_dmamap_load(map->dmat, map, map->buffer, map->len, 1483 map->callback, map->callback_arg, /*flags*/0); 1484 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1485 mtx_lock(&bounce_lock); 1486 } 1487 mtx_unlock(&bounce_lock); 1488} 1489