busdma_machdep.c revision 232356
1/*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * From amd64/busdma_machdep.c, r204214 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/powerpc/powerpc/busdma_machdep.c 232356 2012-03-01 19:58:34Z jhb $"); 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/malloc.h> 37#include <sys/bus.h> 38#include <sys/interrupt.h> 39#include <sys/kernel.h> 40#include <sys/ktr.h> 41#include <sys/lock.h> 42#include <sys/proc.h> 43#include <sys/mutex.h> 44#include <sys/mbuf.h> 45#include <sys/uio.h> 46#include <sys/sysctl.h> 47 48#include <vm/vm.h> 49#include <vm/vm_page.h> 50#include <vm/vm_map.h> 51 52#include <machine/atomic.h> 53#include <machine/bus.h> 54#include <machine/cpufunc.h> 55#include <machine/md_var.h> 56 57#include "iommu_if.h" 58 59#define MAX_BPAGES MIN(8192, physmem/40) 60 61struct bounce_zone; 62 63struct bus_dma_tag { 64 bus_dma_tag_t parent; 65 bus_size_t alignment; 66 bus_addr_t boundary; 67 bus_addr_t lowaddr; 68 bus_addr_t highaddr; 69 bus_dma_filter_t *filter; 70 void *filterarg; 71 bus_size_t maxsize; 72 u_int nsegments; 73 bus_size_t maxsegsz; 74 int flags; 75 int ref_count; 76 int map_count; 77 bus_dma_lock_t *lockfunc; 78 void *lockfuncarg; 79 struct bounce_zone *bounce_zone; 80 device_t iommu; 81 void *iommu_cookie; 82}; 83 84struct bounce_page { 85 vm_offset_t vaddr; /* kva of bounce buffer */ 86 bus_addr_t busaddr; /* Physical address */ 87 vm_offset_t datavaddr; /* kva of client data */ 88 bus_size_t datacount; /* client data count */ 89 STAILQ_ENTRY(bounce_page) links; 90}; 91 92int busdma_swi_pending; 93 94struct bounce_zone { 95 STAILQ_ENTRY(bounce_zone) links; 96 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 97 int total_bpages; 98 int free_bpages; 99 int reserved_bpages; 100 int active_bpages; 101 int total_bounced; 102 int total_deferred; 103 int map_count; 104 bus_size_t alignment; 105 bus_addr_t lowaddr; 106 char zoneid[8]; 107 char lowaddrid[20]; 108 struct sysctl_ctx_list sysctl_tree; 109 struct sysctl_oid *sysctl_tree_top; 110}; 111 112static struct mtx bounce_lock; 113static int total_bpages; 114static int busdma_zonecount; 115static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 116 117static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 118SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 119 "Total bounce pages"); 120 121struct bus_dmamap { 122 struct bp_list bpages; 123 int pagesneeded; 124 int pagesreserved; 125 bus_dma_tag_t dmat; 126 void *buf; /* unmapped buffer pointer */ 127 bus_size_t buflen; /* unmapped buffer length */ 128 bus_dma_segment_t *segments; 129 int nsegs; 130 bus_dmamap_callback_t *callback; 131 void *callback_arg; 132 STAILQ_ENTRY(bus_dmamap) links; 133}; 134 135static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 136static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 137 138static void init_bounce_pages(void *dummy); 139static int alloc_bounce_zone(bus_dma_tag_t dmat); 140static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 141static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 142 int commit); 143static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 144 vm_offset_t vaddr, bus_size_t size); 145static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 146static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 147 148/* 149 * Return true if a match is made. 150 * 151 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 152 * 153 * If paddr is within the bounds of the dma tag then call the filter callback 154 * to check for a match, if there is no filter callback then assume a match. 155 */ 156static __inline int 157run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 158{ 159 int retval; 160 161 retval = 0; 162 163 do { 164 if (dmat->filter == NULL && dmat->iommu == NULL && 165 paddr > dmat->lowaddr && paddr <= dmat->highaddr) 166 retval = 1; 167 if (dmat->filter == NULL && 168 (paddr & (dmat->alignment - 1)) != 0) 169 retval = 1; 170 if (dmat->filter != NULL && 171 (*dmat->filter)(dmat->filterarg, paddr) != 0) 172 retval = 1; 173 174 dmat = dmat->parent; 175 } while (retval == 0 && dmat != NULL); 176 return (retval); 177} 178 179/* 180 * Convenience function for manipulating driver locks from busdma (during 181 * busdma_swi, for example). Drivers that don't provide their own locks 182 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 183 * non-mutex locking scheme don't have to use this at all. 184 */ 185void 186busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 187{ 188 struct mtx *dmtx; 189 190 dmtx = (struct mtx *)arg; 191 switch (op) { 192 case BUS_DMA_LOCK: 193 mtx_lock(dmtx); 194 break; 195 case BUS_DMA_UNLOCK: 196 mtx_unlock(dmtx); 197 break; 198 default: 199 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 200 } 201} 202 203/* 204 * dflt_lock should never get called. It gets put into the dma tag when 205 * lockfunc == NULL, which is only valid if the maps that are associated 206 * with the tag are meant to never be defered. 207 * XXX Should have a way to identify which driver is responsible here. 208 */ 209static void 210dflt_lock(void *arg, bus_dma_lock_op_t op) 211{ 212 panic("driver error: busdma dflt_lock called"); 213} 214 215#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 216#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 217/* 218 * Allocate a device specific dma_tag. 219 */ 220int 221bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 222 bus_addr_t boundary, bus_addr_t lowaddr, 223 bus_addr_t highaddr, bus_dma_filter_t *filter, 224 void *filterarg, bus_size_t maxsize, int nsegments, 225 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 226 void *lockfuncarg, bus_dma_tag_t *dmat) 227{ 228 bus_dma_tag_t newtag; 229 int error = 0; 230 231 /* Basic sanity checking */ 232 if (boundary != 0 && boundary < maxsegsz) 233 maxsegsz = boundary; 234 235 if (maxsegsz == 0) { 236 return (EINVAL); 237 } 238 239 /* Return a NULL tag on failure */ 240 *dmat = NULL; 241 242 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 243 M_ZERO | M_NOWAIT); 244 if (newtag == NULL) { 245 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 246 __func__, newtag, 0, error); 247 return (ENOMEM); 248 } 249 250 newtag->parent = parent; 251 newtag->alignment = alignment; 252 newtag->boundary = boundary; 253 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 254 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 255 newtag->filter = filter; 256 newtag->filterarg = filterarg; 257 newtag->maxsize = maxsize; 258 newtag->nsegments = nsegments; 259 newtag->maxsegsz = maxsegsz; 260 newtag->flags = flags; 261 newtag->ref_count = 1; /* Count ourself */ 262 newtag->map_count = 0; 263 if (lockfunc != NULL) { 264 newtag->lockfunc = lockfunc; 265 newtag->lockfuncarg = lockfuncarg; 266 } else { 267 newtag->lockfunc = dflt_lock; 268 newtag->lockfuncarg = NULL; 269 } 270 271 /* Take into account any restrictions imposed by our parent tag */ 272 if (parent != NULL) { 273 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 274 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 275 if (newtag->boundary == 0) 276 newtag->boundary = parent->boundary; 277 else if (parent->boundary != 0) 278 newtag->boundary = MIN(parent->boundary, 279 newtag->boundary); 280 if (newtag->filter == NULL) { 281 /* 282 * Short circuit looking at our parent directly 283 * since we have encapsulated all of its information 284 */ 285 newtag->filter = parent->filter; 286 newtag->filterarg = parent->filterarg; 287 newtag->parent = parent->parent; 288 } 289 if (newtag->parent != NULL) 290 atomic_add_int(&parent->ref_count, 1); 291 newtag->iommu = parent->iommu; 292 newtag->iommu_cookie = parent->iommu_cookie; 293 } 294 295 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL) 296 newtag->flags |= BUS_DMA_COULD_BOUNCE; 297 298 if (newtag->alignment > 1) 299 newtag->flags |= BUS_DMA_COULD_BOUNCE; 300 301 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 302 (flags & BUS_DMA_ALLOCNOW) != 0) { 303 struct bounce_zone *bz; 304 305 /* Must bounce */ 306 307 if ((error = alloc_bounce_zone(newtag)) != 0) { 308 free(newtag, M_DEVBUF); 309 return (error); 310 } 311 bz = newtag->bounce_zone; 312 313 if (ptoa(bz->total_bpages) < maxsize) { 314 int pages; 315 316 pages = atop(maxsize) - bz->total_bpages; 317 318 /* Add pages to our bounce pool */ 319 if (alloc_bounce_pages(newtag, pages) < pages) 320 error = ENOMEM; 321 } 322 /* Performed initial allocation */ 323 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 324 } 325 326 if (error != 0) { 327 free(newtag, M_DEVBUF); 328 } else { 329 *dmat = newtag; 330 } 331 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 332 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 333 return (error); 334} 335 336int 337bus_dma_tag_destroy(bus_dma_tag_t dmat) 338{ 339 bus_dma_tag_t dmat_copy; 340 int error; 341 342 error = 0; 343 dmat_copy = dmat; 344 345 if (dmat != NULL) { 346 347 if (dmat->map_count != 0) { 348 error = EBUSY; 349 goto out; 350 } 351 352 while (dmat != NULL) { 353 bus_dma_tag_t parent; 354 355 parent = dmat->parent; 356 atomic_subtract_int(&dmat->ref_count, 1); 357 if (dmat->ref_count == 0) { 358 free(dmat, M_DEVBUF); 359 /* 360 * Last reference count, so 361 * release our reference 362 * count on our parent. 363 */ 364 dmat = parent; 365 } else 366 dmat = NULL; 367 } 368 } 369out: 370 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 371 return (error); 372} 373 374/* 375 * Allocate a handle for mapping from kva/uva/physical 376 * address space into bus device space. 377 */ 378int 379bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 380{ 381 int error; 382 383 error = 0; 384 385 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 386 M_NOWAIT | M_ZERO); 387 if (*mapp == NULL) { 388 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 389 __func__, dmat, ENOMEM); 390 return (ENOMEM); 391 } 392 393 394 /* 395 * Bouncing might be required if the driver asks for an active 396 * exclusion region, a data alignment that is stricter than 1, and/or 397 * an active address boundary. 398 */ 399 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 400 401 /* Must bounce */ 402 struct bounce_zone *bz; 403 int maxpages; 404 405 if (dmat->bounce_zone == NULL) { 406 if ((error = alloc_bounce_zone(dmat)) != 0) 407 return (error); 408 } 409 bz = dmat->bounce_zone; 410 411 /* Initialize the new map */ 412 STAILQ_INIT(&((*mapp)->bpages)); 413 414 /* 415 * Attempt to add pages to our pool on a per-instance 416 * basis up to a sane limit. 417 */ 418 if (dmat->alignment > 1) 419 maxpages = MAX_BPAGES; 420 else 421 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); 422 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 423 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 424 int pages; 425 426 pages = MAX(atop(dmat->maxsize), 1); 427 pages = MIN(maxpages - bz->total_bpages, pages); 428 pages = MAX(pages, 1); 429 if (alloc_bounce_pages(dmat, pages) < pages) 430 error = ENOMEM; 431 432 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 433 if (error == 0) 434 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 435 } else { 436 error = 0; 437 } 438 } 439 bz->map_count++; 440 } 441 442 (*mapp)->nsegs = 0; 443 (*mapp)->segments = (bus_dma_segment_t *)malloc( 444 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 445 M_NOWAIT); 446 if ((*mapp)->segments == NULL) { 447 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 448 __func__, dmat, ENOMEM); 449 return (ENOMEM); 450 } 451 452 if (error == 0) 453 dmat->map_count++; 454 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 455 __func__, dmat, dmat->flags, error); 456 return (error); 457} 458 459/* 460 * Destroy a handle for mapping from kva/uva/physical 461 * address space into bus device space. 462 */ 463int 464bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 465{ 466 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 467 if (STAILQ_FIRST(&map->bpages) != NULL) { 468 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 469 __func__, dmat, EBUSY); 470 return (EBUSY); 471 } 472 if (dmat->bounce_zone) 473 dmat->bounce_zone->map_count--; 474 } 475 free(map->segments, M_DEVBUF); 476 free(map, M_DEVBUF); 477 dmat->map_count--; 478 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 479 return (0); 480} 481 482 483/* 484 * Allocate a piece of memory that can be efficiently mapped into 485 * bus device space based on the constraints lited in the dma tag. 486 * A dmamap to for use with dmamap_load is also allocated. 487 */ 488int 489bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 490 bus_dmamap_t *mapp) 491{ 492 int mflags; 493 494 if (flags & BUS_DMA_NOWAIT) 495 mflags = M_NOWAIT; 496 else 497 mflags = M_WAITOK; 498 499 bus_dmamap_create(dmat, flags, mapp); 500 501 if (flags & BUS_DMA_ZERO) 502 mflags |= M_ZERO; 503 504 /* 505 * XXX: 506 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact 507 * alignment guarantees of malloc need to be nailed down, and the 508 * code below should be rewritten to take that into account. 509 * 510 * In the meantime, we'll warn the user if malloc gets it wrong. 511 */ 512 if ((dmat->maxsize <= PAGE_SIZE) && 513 (dmat->alignment < dmat->maxsize) && 514 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 515 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 516 } else { 517 /* 518 * XXX Use Contigmalloc until it is merged into this facility 519 * and handles multi-seg allocations. Nobody is doing 520 * multi-seg allocations yet though. 521 * XXX Certain AGP hardware does. 522 */ 523 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 524 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 525 dmat->boundary); 526 } 527 if (*vaddr == NULL) { 528 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 529 __func__, dmat, dmat->flags, ENOMEM); 530 return (ENOMEM); 531 } else if (vtophys(*vaddr) & (dmat->alignment - 1)) { 532 printf("bus_dmamem_alloc failed to align memory properly.\n"); 533 } 534#ifdef NOTYET 535 if (flags & BUS_DMA_NOCACHE) 536 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize, 537 VM_MEMATTR_UNCACHEABLE); 538#endif 539 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 540 __func__, dmat, dmat->flags, 0); 541 return (0); 542} 543 544/* 545 * Free a piece of memory and it's allociated dmamap, that was allocated 546 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 547 */ 548void 549bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 550{ 551 bus_dmamap_destroy(dmat, map); 552 553#ifdef NOTYET 554 pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, VM_MEMATTR_DEFAULT); 555#endif 556 if ((dmat->maxsize <= PAGE_SIZE) && 557 (dmat->alignment < dmat->maxsize) && 558 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 559 free(vaddr, M_DEVBUF); 560 else { 561 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 562 } 563 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 564} 565 566/* 567 * Utility function to load a linear buffer. lastaddrp holds state 568 * between invocations (for multiple-buffer loads). segp contains 569 * the starting segment on entrance, and the ending segment on exit. 570 * first indicates if this is the first invocation of this function. 571 */ 572static __inline int 573_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 574 bus_dmamap_t map, 575 void *buf, bus_size_t buflen, 576 pmap_t pmap, 577 int flags, 578 bus_addr_t *lastaddrp, 579 bus_dma_segment_t *segs, 580 int *segp, 581 int first) 582{ 583 bus_size_t sgsize; 584 bus_addr_t curaddr, lastaddr, baddr, bmask; 585 vm_offset_t vaddr; 586 bus_addr_t paddr; 587 int seg; 588 589 if (map->pagesneeded == 0 && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { 590 vm_offset_t vendaddr; 591 592 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 593 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), 594 dmat->boundary, dmat->alignment); 595 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); 596 /* 597 * Count the number of bounce pages 598 * needed in order to complete this transfer 599 */ 600 vaddr = (vm_offset_t)buf; 601 vendaddr = (vm_offset_t)buf + buflen; 602 603 while (vaddr < vendaddr) { 604 bus_size_t sg_len; 605 606 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 607 if (pmap) 608 paddr = pmap_extract(pmap, vaddr); 609 else 610 paddr = pmap_kextract(vaddr); 611 if (run_filter(dmat, paddr) != 0) { 612 sg_len = roundup2(sg_len, dmat->alignment); 613 map->pagesneeded++; 614 } 615 vaddr += sg_len; 616 } 617 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 618 } 619 620 /* Reserve Necessary Bounce Pages */ 621 if (map->pagesneeded != 0) { 622 mtx_lock(&bounce_lock); 623 if (flags & BUS_DMA_NOWAIT) { 624 if (reserve_bounce_pages(dmat, map, 0) != 0) { 625 mtx_unlock(&bounce_lock); 626 return (ENOMEM); 627 } 628 } else { 629 if (reserve_bounce_pages(dmat, map, 1) != 0) { 630 /* Queue us for resources */ 631 map->dmat = dmat; 632 map->buf = buf; 633 map->buflen = buflen; 634 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 635 map, links); 636 mtx_unlock(&bounce_lock); 637 return (EINPROGRESS); 638 } 639 } 640 mtx_unlock(&bounce_lock); 641 } 642 643 vaddr = (vm_offset_t)buf; 644 lastaddr = *lastaddrp; 645 bmask = ~(dmat->boundary - 1); 646 647 for (seg = *segp; buflen > 0 ; ) { 648 bus_size_t max_sgsize; 649 650 /* 651 * Get the physical address for this segment. 652 */ 653 if (pmap) 654 curaddr = pmap_extract(pmap, vaddr); 655 else 656 curaddr = pmap_kextract(vaddr); 657 658 /* 659 * Compute the segment size, and adjust counts. 660 */ 661 max_sgsize = MIN(buflen, dmat->maxsegsz); 662 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK); 663 if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 664 sgsize = roundup2(sgsize, dmat->alignment); 665 sgsize = MIN(sgsize, max_sgsize); 666 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 667 } else { 668 sgsize = MIN(sgsize, max_sgsize); 669 } 670 671 /* 672 * Make sure we don't cross any boundaries. 673 */ 674 if (dmat->boundary > 0) { 675 baddr = (curaddr + dmat->boundary) & bmask; 676 if (sgsize > (baddr - curaddr)) 677 sgsize = (baddr - curaddr); 678 } 679 680 /* 681 * Insert chunk into a segment, coalescing with 682 * previous segment if possible. 683 */ 684 if (first) { 685 segs[seg].ds_addr = curaddr; 686 segs[seg].ds_len = sgsize; 687 first = 0; 688 } else { 689 if (curaddr == lastaddr && 690 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 691 (dmat->boundary == 0 || 692 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 693 segs[seg].ds_len += sgsize; 694 else { 695 if (++seg >= dmat->nsegments) 696 break; 697 segs[seg].ds_addr = curaddr; 698 segs[seg].ds_len = sgsize; 699 } 700 } 701 702 lastaddr = curaddr + sgsize; 703 vaddr += sgsize; 704 buflen -= sgsize; 705 } 706 707 *segp = seg; 708 *lastaddrp = lastaddr; 709 710 /* 711 * Did we fit? 712 */ 713 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 714} 715 716/* 717 * Map the buffer buf into bus space using the dmamap map. 718 */ 719int 720bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 721 bus_size_t buflen, bus_dmamap_callback_t *callback, 722 void *callback_arg, int flags) 723{ 724 bus_addr_t lastaddr = 0; 725 int error; 726 727 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 728 flags |= BUS_DMA_WAITOK; 729 map->callback = callback; 730 map->callback_arg = callback_arg; 731 } 732 733 map->nsegs = 0; 734 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 735 &lastaddr, map->segments, &map->nsegs, 1); 736 map->nsegs++; 737 738 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 739 __func__, dmat, dmat->flags, error, map->nsegs); 740 741 if (error == EINPROGRESS) { 742 return (error); 743 } 744 745 if (dmat->iommu != NULL) 746 IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs, dmat->lowaddr, 747 dmat->highaddr, dmat->alignment, dmat->boundary, 748 dmat->iommu_cookie); 749 750 if (error) 751 (*callback)(callback_arg, map->segments, 0, error); 752 else 753 (*callback)(callback_arg, map->segments, map->nsegs, 0); 754 755 /* 756 * Return ENOMEM to the caller so that it can pass it up the stack. 757 * This error only happens when NOWAIT is set, so deferal is disabled. 758 */ 759 if (error == ENOMEM) 760 return (error); 761 762 return (0); 763} 764 765 766/* 767 * Like _bus_dmamap_load(), but for mbufs. 768 */ 769int 770bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 771 struct mbuf *m0, 772 bus_dmamap_callback2_t *callback, void *callback_arg, 773 int flags) 774{ 775 int error; 776 777 M_ASSERTPKTHDR(m0); 778 779 flags |= BUS_DMA_NOWAIT; 780 map->nsegs = 0; 781 error = 0; 782 if (m0->m_pkthdr.len <= dmat->maxsize) { 783 int first = 1; 784 bus_addr_t lastaddr = 0; 785 struct mbuf *m; 786 787 for (m = m0; m != NULL && error == 0; m = m->m_next) { 788 if (m->m_len > 0) { 789 error = _bus_dmamap_load_buffer(dmat, map, 790 m->m_data, m->m_len, 791 NULL, flags, &lastaddr, 792 map->segments, &map->nsegs, first); 793 first = 0; 794 } 795 } 796 } else { 797 error = EINVAL; 798 } 799 800 map->nsegs++; 801 if (dmat->iommu != NULL) 802 IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs, dmat->lowaddr, 803 dmat->highaddr, dmat->alignment, dmat->boundary, 804 dmat->iommu_cookie); 805 806 if (error) { 807 /* force "no valid mappings" in callback */ 808 (*callback)(callback_arg, map->segments, 0, 0, error); 809 } else { 810 (*callback)(callback_arg, map->segments, 811 map->nsegs, m0->m_pkthdr.len, error); 812 } 813 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 814 __func__, dmat, dmat->flags, error, map->nsegs); 815 return (error); 816} 817 818int 819bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 820 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 821 int flags) 822{ 823 int error; 824 825 M_ASSERTPKTHDR(m0); 826 827 flags |= BUS_DMA_NOWAIT; 828 *nsegs = 0; 829 error = 0; 830 if (m0->m_pkthdr.len <= dmat->maxsize) { 831 int first = 1; 832 bus_addr_t lastaddr = 0; 833 struct mbuf *m; 834 835 for (m = m0; m != NULL && error == 0; m = m->m_next) { 836 if (m->m_len > 0) { 837 error = _bus_dmamap_load_buffer(dmat, map, 838 m->m_data, m->m_len, 839 NULL, flags, &lastaddr, 840 segs, nsegs, first); 841 first = 0; 842 } 843 } 844 } else { 845 error = EINVAL; 846 } 847 848 /* XXX FIXME: Having to increment nsegs is really annoying */ 849 ++*nsegs; 850 851 if (dmat->iommu != NULL) 852 IOMMU_MAP(dmat->iommu, segs, nsegs, dmat->lowaddr, 853 dmat->highaddr, dmat->alignment, dmat->boundary, 854 dmat->iommu_cookie); 855 856 map->nsegs = *nsegs; 857 memcpy(map->segments, segs, map->nsegs*sizeof(segs[0])); 858 859 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 860 __func__, dmat, dmat->flags, error, *nsegs); 861 return (error); 862} 863 864/* 865 * Like _bus_dmamap_load(), but for uios. 866 */ 867int 868bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 869 struct uio *uio, 870 bus_dmamap_callback2_t *callback, void *callback_arg, 871 int flags) 872{ 873 bus_addr_t lastaddr = 0; 874 int error, first, i; 875 bus_size_t resid; 876 struct iovec *iov; 877 pmap_t pmap; 878 879 flags |= BUS_DMA_NOWAIT; 880 resid = uio->uio_resid; 881 iov = uio->uio_iov; 882 883 if (uio->uio_segflg == UIO_USERSPACE) { 884 KASSERT(uio->uio_td != NULL, 885 ("bus_dmamap_load_uio: USERSPACE but no proc")); 886 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 887 } else 888 pmap = NULL; 889 890 map->nsegs = 0; 891 error = 0; 892 first = 1; 893 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 894 /* 895 * Now at the first iovec to load. Load each iovec 896 * until we have exhausted the residual count. 897 */ 898 bus_size_t minlen = 899 resid < iov[i].iov_len ? resid : iov[i].iov_len; 900 caddr_t addr = (caddr_t) iov[i].iov_base; 901 902 if (minlen > 0) { 903 error = _bus_dmamap_load_buffer(dmat, map, 904 addr, minlen, pmap, flags, &lastaddr, 905 map->segments, &map->nsegs, first); 906 first = 0; 907 908 resid -= minlen; 909 } 910 } 911 912 map->nsegs++; 913 if (dmat->iommu != NULL) 914 IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs, dmat->lowaddr, 915 dmat->highaddr, dmat->alignment, dmat->boundary, 916 dmat->iommu_cookie); 917 918 if (error) { 919 /* force "no valid mappings" in callback */ 920 (*callback)(callback_arg, map->segments, 0, 0, error); 921 } else { 922 (*callback)(callback_arg, map->segments, 923 map->nsegs, uio->uio_resid, error); 924 } 925 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 926 __func__, dmat, dmat->flags, error, map->nsegs); 927 return (error); 928} 929 930/* 931 * Release the mapping held by map. 932 */ 933void 934_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 935{ 936 struct bounce_page *bpage; 937 938 if (dmat->iommu) { 939 IOMMU_UNMAP(dmat->iommu, map->segments, map->nsegs, dmat->iommu_cookie); 940 map->nsegs = 0; 941 } 942 943 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 944 STAILQ_REMOVE_HEAD(&map->bpages, links); 945 free_bounce_page(dmat, bpage); 946 } 947} 948 949void 950_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 951{ 952 struct bounce_page *bpage; 953 954 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 955 /* 956 * Handle data bouncing. We might also 957 * want to add support for invalidating 958 * the caches on broken hardware 959 */ 960 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 961 "performing bounce", __func__, op, dmat, dmat->flags); 962 963 if (op & BUS_DMASYNC_PREWRITE) { 964 while (bpage != NULL) { 965 bcopy((void *)bpage->datavaddr, 966 (void *)bpage->vaddr, 967 bpage->datacount); 968 bpage = STAILQ_NEXT(bpage, links); 969 } 970 dmat->bounce_zone->total_bounced++; 971 } 972 973 if (op & BUS_DMASYNC_POSTREAD) { 974 while (bpage != NULL) { 975 bcopy((void *)bpage->vaddr, 976 (void *)bpage->datavaddr, 977 bpage->datacount); 978 bpage = STAILQ_NEXT(bpage, links); 979 } 980 dmat->bounce_zone->total_bounced++; 981 } 982 } 983 984 powerpc_sync(); 985} 986 987static void 988init_bounce_pages(void *dummy __unused) 989{ 990 991 total_bpages = 0; 992 STAILQ_INIT(&bounce_zone_list); 993 STAILQ_INIT(&bounce_map_waitinglist); 994 STAILQ_INIT(&bounce_map_callbacklist); 995 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 996} 997SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 998 999static struct sysctl_ctx_list * 1000busdma_sysctl_tree(struct bounce_zone *bz) 1001{ 1002 return (&bz->sysctl_tree); 1003} 1004 1005static struct sysctl_oid * 1006busdma_sysctl_tree_top(struct bounce_zone *bz) 1007{ 1008 return (bz->sysctl_tree_top); 1009} 1010 1011static int 1012alloc_bounce_zone(bus_dma_tag_t dmat) 1013{ 1014 struct bounce_zone *bz; 1015 1016 /* Check to see if we already have a suitable zone */ 1017 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1018 if ((dmat->alignment <= bz->alignment) 1019 && (dmat->lowaddr >= bz->lowaddr)) { 1020 dmat->bounce_zone = bz; 1021 return (0); 1022 } 1023 } 1024 1025 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1026 M_NOWAIT | M_ZERO)) == NULL) 1027 return (ENOMEM); 1028 1029 STAILQ_INIT(&bz->bounce_page_list); 1030 bz->free_bpages = 0; 1031 bz->reserved_bpages = 0; 1032 bz->active_bpages = 0; 1033 bz->lowaddr = dmat->lowaddr; 1034 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1035 bz->map_count = 0; 1036 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1037 busdma_zonecount++; 1038 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1039 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1040 dmat->bounce_zone = bz; 1041 1042 sysctl_ctx_init(&bz->sysctl_tree); 1043 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1044 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1045 CTLFLAG_RD, 0, ""); 1046 if (bz->sysctl_tree_top == NULL) { 1047 sysctl_ctx_free(&bz->sysctl_tree); 1048 return (0); /* XXX error code? */ 1049 } 1050 1051 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1052 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1053 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1054 "Total bounce pages"); 1055 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1056 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1057 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1058 "Free bounce pages"); 1059 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1060 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1061 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1062 "Reserved bounce pages"); 1063 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1064 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1065 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1066 "Active bounce pages"); 1067 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1068 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1069 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1070 "Total bounce requests"); 1071 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1072 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1073 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1074 "Total bounce requests that were deferred"); 1075 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1076 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1077 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1078 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1079 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1080 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1081 1082 return (0); 1083} 1084 1085static int 1086alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1087{ 1088 struct bounce_zone *bz; 1089 int count; 1090 1091 bz = dmat->bounce_zone; 1092 count = 0; 1093 while (numpages > 0) { 1094 struct bounce_page *bpage; 1095 1096 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1097 M_NOWAIT | M_ZERO); 1098 1099 if (bpage == NULL) 1100 break; 1101 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1102 M_NOWAIT, 0ul, 1103 bz->lowaddr, 1104 PAGE_SIZE, 1105 0); 1106 if (bpage->vaddr == 0) { 1107 free(bpage, M_DEVBUF); 1108 break; 1109 } 1110 bpage->busaddr = pmap_kextract(bpage->vaddr); 1111 mtx_lock(&bounce_lock); 1112 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1113 total_bpages++; 1114 bz->total_bpages++; 1115 bz->free_bpages++; 1116 mtx_unlock(&bounce_lock); 1117 count++; 1118 numpages--; 1119 } 1120 return (count); 1121} 1122 1123static int 1124reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1125{ 1126 struct bounce_zone *bz; 1127 int pages; 1128 1129 mtx_assert(&bounce_lock, MA_OWNED); 1130 bz = dmat->bounce_zone; 1131 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1132 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1133 return (map->pagesneeded - (map->pagesreserved + pages)); 1134 bz->free_bpages -= pages; 1135 bz->reserved_bpages += pages; 1136 map->pagesreserved += pages; 1137 pages = map->pagesneeded - map->pagesreserved; 1138 1139 return (pages); 1140} 1141 1142static bus_addr_t 1143add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1144 bus_size_t size) 1145{ 1146 struct bounce_zone *bz; 1147 struct bounce_page *bpage; 1148 1149 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1150 1151 bz = dmat->bounce_zone; 1152 if (map->pagesneeded == 0) 1153 panic("add_bounce_page: map doesn't need any pages"); 1154 map->pagesneeded--; 1155 1156 if (map->pagesreserved == 0) 1157 panic("add_bounce_page: map doesn't need any pages"); 1158 map->pagesreserved--; 1159 1160 mtx_lock(&bounce_lock); 1161 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1162 if (bpage == NULL) 1163 panic("add_bounce_page: free page list is empty"); 1164 1165 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1166 bz->reserved_bpages--; 1167 bz->active_bpages++; 1168 mtx_unlock(&bounce_lock); 1169 1170 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1171 /* Page offset needs to be preserved. */ 1172 bpage->vaddr |= vaddr & PAGE_MASK; 1173 bpage->busaddr |= vaddr & PAGE_MASK; 1174 } 1175 bpage->datavaddr = vaddr; 1176 bpage->datacount = size; 1177 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1178 return (bpage->busaddr); 1179} 1180 1181static void 1182free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1183{ 1184 struct bus_dmamap *map; 1185 struct bounce_zone *bz; 1186 1187 bz = dmat->bounce_zone; 1188 bpage->datavaddr = 0; 1189 bpage->datacount = 0; 1190 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1191 /* 1192 * Reset the bounce page to start at offset 0. Other uses 1193 * of this bounce page may need to store a full page of 1194 * data and/or assume it starts on a page boundary. 1195 */ 1196 bpage->vaddr &= ~PAGE_MASK; 1197 bpage->busaddr &= ~PAGE_MASK; 1198 } 1199 1200 mtx_lock(&bounce_lock); 1201 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1202 bz->free_bpages++; 1203 bz->active_bpages--; 1204 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1205 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1206 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1207 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1208 map, links); 1209 busdma_swi_pending = 1; 1210 bz->total_deferred++; 1211 swi_sched(vm_ih, 0); 1212 } 1213 } 1214 mtx_unlock(&bounce_lock); 1215} 1216 1217void 1218busdma_swi(void) 1219{ 1220 bus_dma_tag_t dmat; 1221 struct bus_dmamap *map; 1222 1223 mtx_lock(&bounce_lock); 1224 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1225 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1226 mtx_unlock(&bounce_lock); 1227 dmat = map->dmat; 1228 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1229 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1230 map->callback, map->callback_arg, /*flags*/0); 1231 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1232 mtx_lock(&bounce_lock); 1233 } 1234 mtx_unlock(&bounce_lock); 1235} 1236 1237int 1238bus_dma_tag_set_iommu(bus_dma_tag_t tag, struct device *iommu, void *cookie) 1239{ 1240 tag->iommu = iommu; 1241 tag->iommu_cookie = cookie; 1242 1243 return (0); 1244} 1245 1246