busdma_machdep.c revision 232267
1/*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/x86/x86/busdma_machdep.c 232267 2012-02-28 19:42:40Z emaste $"); 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/malloc.h> 33#include <sys/bus.h> 34#include <sys/interrupt.h> 35#include <sys/kernel.h> 36#include <sys/ktr.h> 37#include <sys/lock.h> 38#include <sys/proc.h> 39#include <sys/mutex.h> 40#include <sys/mbuf.h> 41#include <sys/uio.h> 42#include <sys/sysctl.h> 43 44#include <vm/vm.h> 45#include <vm/vm_page.h> 46#include <vm/vm_map.h> 47 48#include <machine/atomic.h> 49#include <machine/bus.h> 50#include <machine/md_var.h> 51#include <machine/specialreg.h> 52 53#ifdef __i386__ 54#define MAX_BPAGES 512 55#else 56#define MAX_BPAGES 8192 57#endif 58#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 59#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 60 61struct bounce_zone; 62 63struct bus_dma_tag { 64 bus_dma_tag_t parent; 65 bus_size_t alignment; 66 bus_size_t boundary; 67 bus_addr_t lowaddr; 68 bus_addr_t highaddr; 69 bus_dma_filter_t *filter; 70 void *filterarg; 71 bus_size_t maxsize; 72 u_int nsegments; 73 bus_size_t maxsegsz; 74 int flags; 75 int ref_count; 76 int map_count; 77 bus_dma_lock_t *lockfunc; 78 void *lockfuncarg; 79 bus_dma_segment_t *segments; 80 struct bounce_zone *bounce_zone; 81}; 82 83struct bounce_page { 84 vm_offset_t vaddr; /* kva of bounce buffer */ 85 bus_addr_t busaddr; /* Physical address */ 86 vm_offset_t datavaddr; /* kva of client data */ 87 bus_size_t datacount; /* client data count */ 88 STAILQ_ENTRY(bounce_page) links; 89}; 90 91int busdma_swi_pending; 92 93struct bounce_zone { 94 STAILQ_ENTRY(bounce_zone) links; 95 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 96 int total_bpages; 97 int free_bpages; 98 int reserved_bpages; 99 int active_bpages; 100 int total_bounced; 101 int total_deferred; 102 int map_count; 103 bus_size_t alignment; 104 bus_addr_t lowaddr; 105 char zoneid[8]; 106 char lowaddrid[20]; 107 struct sysctl_ctx_list sysctl_tree; 108 struct sysctl_oid *sysctl_tree_top; 109}; 110 111static struct mtx bounce_lock; 112static int total_bpages; 113static int busdma_zonecount; 114static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 115 116static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 117SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 118 "Total bounce pages"); 119 120struct bus_dmamap { 121 struct bp_list bpages; 122 int pagesneeded; 123 int pagesreserved; 124 bus_dma_tag_t dmat; 125 void *buf; /* unmapped buffer pointer */ 126 bus_size_t buflen; /* unmapped buffer length */ 127 bus_dmamap_callback_t *callback; 128 void *callback_arg; 129 STAILQ_ENTRY(bus_dmamap) links; 130}; 131 132static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 133static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 134static struct bus_dmamap nobounce_dmamap; 135 136static void init_bounce_pages(void *dummy); 137static int alloc_bounce_zone(bus_dma_tag_t dmat); 138static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 139static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 140 int commit); 141static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 142 vm_offset_t vaddr, bus_size_t size); 143static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 144int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 145int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 146 void *buf, bus_size_t buflen, int flags); 147 148#ifdef XEN 149#undef pmap_kextract 150#define pmap_kextract pmap_kextract_ma 151#endif 152 153/* 154 * Return true if a match is made. 155 * 156 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 157 * 158 * If paddr is within the bounds of the dma tag then call the filter callback 159 * to check for a match, if there is no filter callback then assume a match. 160 */ 161int 162run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 163{ 164 int retval; 165 166 retval = 0; 167 168 do { 169 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 170 || ((paddr & (dmat->alignment - 1)) != 0)) 171 && (dmat->filter == NULL 172 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 173 retval = 1; 174 175 dmat = dmat->parent; 176 } while (retval == 0 && dmat != NULL); 177 return (retval); 178} 179 180/* 181 * Convenience function for manipulating driver locks from busdma (during 182 * busdma_swi, for example). Drivers that don't provide their own locks 183 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 184 * non-mutex locking scheme don't have to use this at all. 185 */ 186void 187busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 188{ 189 struct mtx *dmtx; 190 191 dmtx = (struct mtx *)arg; 192 switch (op) { 193 case BUS_DMA_LOCK: 194 mtx_lock(dmtx); 195 break; 196 case BUS_DMA_UNLOCK: 197 mtx_unlock(dmtx); 198 break; 199 default: 200 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 201 } 202} 203 204/* 205 * dflt_lock should never get called. It gets put into the dma tag when 206 * lockfunc == NULL, which is only valid if the maps that are associated 207 * with the tag are meant to never be defered. 208 * XXX Should have a way to identify which driver is responsible here. 209 */ 210static void 211dflt_lock(void *arg, bus_dma_lock_op_t op) 212{ 213 panic("driver error: busdma dflt_lock called"); 214} 215 216/* 217 * Allocate a device specific dma_tag. 218 */ 219int 220bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 221 bus_size_t boundary, bus_addr_t lowaddr, 222 bus_addr_t highaddr, bus_dma_filter_t *filter, 223 void *filterarg, bus_size_t maxsize, int nsegments, 224 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 225 void *lockfuncarg, bus_dma_tag_t *dmat) 226{ 227 bus_dma_tag_t newtag; 228 int error = 0; 229 230 /* Always enforce at least a 4GB (2GB for PAE) boundary. */ 231#if defined(__amd64__) 232 if (boundary == 0 || boundary > ((bus_addr_t)1 << 32)) 233 boundary = (bus_size_t)1 << 32; 234#elif defined(PAE) 235 if (boundary == 0 || boundary > ((bus_addr_t)1 << 31)) 236 boundary = (bus_size_t)1 << 31; 237#endif 238 /* Basic sanity checking */ 239 if (boundary != 0 && boundary < maxsegsz) 240 maxsegsz = boundary; 241 242 if (maxsegsz == 0) { 243 return (EINVAL); 244 } 245 246 /* Return a NULL tag on failure */ 247 *dmat = NULL; 248 249 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 250 M_ZERO | M_NOWAIT); 251 if (newtag == NULL) { 252 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 253 __func__, newtag, 0, error); 254 return (ENOMEM); 255 } 256 257 newtag->parent = parent; 258 newtag->alignment = alignment; 259 newtag->boundary = boundary; 260 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 261 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 262 newtag->filter = filter; 263 newtag->filterarg = filterarg; 264 newtag->maxsize = maxsize; 265 newtag->nsegments = nsegments; 266 newtag->maxsegsz = maxsegsz; 267 newtag->flags = flags; 268 newtag->ref_count = 1; /* Count ourself */ 269 newtag->map_count = 0; 270 if (lockfunc != NULL) { 271 newtag->lockfunc = lockfunc; 272 newtag->lockfuncarg = lockfuncarg; 273 } else { 274 newtag->lockfunc = dflt_lock; 275 newtag->lockfuncarg = NULL; 276 } 277 newtag->segments = NULL; 278 279 /* Take into account any restrictions imposed by our parent tag */ 280 if (parent != NULL) { 281 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 282 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 283 if (newtag->boundary == 0) 284 newtag->boundary = parent->boundary; 285 else if (parent->boundary != 0) 286 newtag->boundary = MIN(parent->boundary, 287 newtag->boundary); 288 if ((newtag->filter != NULL) || 289 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 290 newtag->flags |= BUS_DMA_COULD_BOUNCE; 291 if (newtag->filter == NULL) { 292 /* 293 * Short circuit looking at our parent directly 294 * since we have encapsulated all of its information 295 */ 296 newtag->filter = parent->filter; 297 newtag->filterarg = parent->filterarg; 298 newtag->parent = parent->parent; 299 } 300 if (newtag->parent != NULL) 301 atomic_add_int(&parent->ref_count, 1); 302 } 303 304 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) 305 || newtag->alignment > 1) 306 newtag->flags |= BUS_DMA_COULD_BOUNCE; 307 308 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 309 (flags & BUS_DMA_ALLOCNOW) != 0) { 310 struct bounce_zone *bz; 311 312 /* Must bounce */ 313 314 if ((error = alloc_bounce_zone(newtag)) != 0) { 315 free(newtag, M_DEVBUF); 316 return (error); 317 } 318 bz = newtag->bounce_zone; 319 320 if (ptoa(bz->total_bpages) < maxsize) { 321 int pages; 322 323 pages = atop(maxsize) - bz->total_bpages; 324 325 /* Add pages to our bounce pool */ 326 if (alloc_bounce_pages(newtag, pages) < pages) 327 error = ENOMEM; 328 } 329 /* Performed initial allocation */ 330 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 331 } 332 333 if (error != 0) { 334 free(newtag, M_DEVBUF); 335 } else { 336 *dmat = newtag; 337 } 338 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 339 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 340 return (error); 341} 342 343int 344bus_dma_tag_destroy(bus_dma_tag_t dmat) 345{ 346 bus_dma_tag_t dmat_copy; 347 int error; 348 349 error = 0; 350 dmat_copy = dmat; 351 352 if (dmat != NULL) { 353 354 if (dmat->map_count != 0) { 355 error = EBUSY; 356 goto out; 357 } 358 359 while (dmat != NULL) { 360 bus_dma_tag_t parent; 361 362 parent = dmat->parent; 363 atomic_subtract_int(&dmat->ref_count, 1); 364 if (dmat->ref_count == 0) { 365 if (dmat->segments != NULL) 366 free(dmat->segments, M_DEVBUF); 367 free(dmat, M_DEVBUF); 368 /* 369 * Last reference count, so 370 * release our reference 371 * count on our parent. 372 */ 373 dmat = parent; 374 } else 375 dmat = NULL; 376 } 377 } 378out: 379 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 380 return (error); 381} 382 383/* 384 * Allocate a handle for mapping from kva/uva/physical 385 * address space into bus device space. 386 */ 387int 388bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 389{ 390 int error; 391 392 error = 0; 393 394 if (dmat->segments == NULL) { 395 dmat->segments = (bus_dma_segment_t *)malloc( 396 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 397 M_NOWAIT); 398 if (dmat->segments == NULL) { 399 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 400 __func__, dmat, ENOMEM); 401 return (ENOMEM); 402 } 403 } 404 405 /* 406 * Bouncing might be required if the driver asks for an active 407 * exclusion region, a data alignment that is stricter than 1, and/or 408 * an active address boundary. 409 */ 410 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 411 412 /* Must bounce */ 413 struct bounce_zone *bz; 414 int maxpages; 415 416 if (dmat->bounce_zone == NULL) { 417 if ((error = alloc_bounce_zone(dmat)) != 0) 418 return (error); 419 } 420 bz = dmat->bounce_zone; 421 422 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 423 M_NOWAIT | M_ZERO); 424 if (*mapp == NULL) { 425 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 426 __func__, dmat, ENOMEM); 427 return (ENOMEM); 428 } 429 430 /* Initialize the new map */ 431 STAILQ_INIT(&((*mapp)->bpages)); 432 433 /* 434 * Attempt to add pages to our pool on a per-instance 435 * basis up to a sane limit. 436 */ 437 if (dmat->alignment > 1) 438 maxpages = MAX_BPAGES; 439 else 440 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); 441 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 442 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 443 int pages; 444 445 pages = MAX(atop(dmat->maxsize), 1); 446 pages = MIN(maxpages - bz->total_bpages, pages); 447 pages = MAX(pages, 1); 448 if (alloc_bounce_pages(dmat, pages) < pages) 449 error = ENOMEM; 450 451 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 452 if (error == 0) 453 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 454 } else { 455 error = 0; 456 } 457 } 458 bz->map_count++; 459 } else { 460 *mapp = NULL; 461 } 462 if (error == 0) 463 dmat->map_count++; 464 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 465 __func__, dmat, dmat->flags, error); 466 return (error); 467} 468 469/* 470 * Destroy a handle for mapping from kva/uva/physical 471 * address space into bus device space. 472 */ 473int 474bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 475{ 476 if (map != NULL && map != &nobounce_dmamap) { 477 if (STAILQ_FIRST(&map->bpages) != NULL) { 478 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 479 __func__, dmat, EBUSY); 480 return (EBUSY); 481 } 482 if (dmat->bounce_zone) 483 dmat->bounce_zone->map_count--; 484 free(map, M_DEVBUF); 485 } 486 dmat->map_count--; 487 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 488 return (0); 489} 490 491 492/* 493 * Allocate a piece of memory that can be efficiently mapped into 494 * bus device space based on the constraints lited in the dma tag. 495 * A dmamap to for use with dmamap_load is also allocated. 496 */ 497int 498bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 499 bus_dmamap_t *mapp) 500{ 501 int mflags; 502 503 if (flags & BUS_DMA_NOWAIT) 504 mflags = M_NOWAIT; 505 else 506 mflags = M_WAITOK; 507 508 /* If we succeed, no mapping/bouncing will be required */ 509 *mapp = NULL; 510 511 if (dmat->segments == NULL) { 512 dmat->segments = (bus_dma_segment_t *)malloc( 513 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 514 mflags); 515 if (dmat->segments == NULL) { 516 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 517 __func__, dmat, dmat->flags, ENOMEM); 518 return (ENOMEM); 519 } 520 } 521 if (flags & BUS_DMA_ZERO) 522 mflags |= M_ZERO; 523 524 /* 525 * XXX: 526 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact 527 * alignment guarantees of malloc need to be nailed down, and the 528 * code below should be rewritten to take that into account. 529 * 530 * In the meantime, we'll warn the user if malloc gets it wrong. 531 */ 532 if ((dmat->maxsize <= PAGE_SIZE) && 533 (dmat->alignment < dmat->maxsize) && 534 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 535 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 536 } else { 537 /* 538 * XXX Use Contigmalloc until it is merged into this facility 539 * and handles multi-seg allocations. Nobody is doing 540 * multi-seg allocations yet though. 541 * XXX Certain AGP hardware does. 542 */ 543 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 544 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 545 dmat->boundary); 546 } 547 if (*vaddr == NULL) { 548 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 549 __func__, dmat, dmat->flags, ENOMEM); 550 return (ENOMEM); 551 } else if (vtophys(*vaddr) & (dmat->alignment - 1)) { 552 printf("bus_dmamem_alloc failed to align memory properly.\n"); 553 } 554 if (flags & BUS_DMA_NOCACHE) 555 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize, 556 PAT_UNCACHEABLE); 557 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 558 __func__, dmat, dmat->flags, 0); 559 return (0); 560} 561 562/* 563 * Free a piece of memory and it's allociated dmamap, that was allocated 564 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 565 */ 566void 567bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 568{ 569 /* 570 * dmamem does not need to be bounced, so the map should be 571 * NULL 572 */ 573 if (map != NULL) 574 panic("bus_dmamem_free: Invalid map freed\n"); 575 pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK); 576 if ((dmat->maxsize <= PAGE_SIZE) && 577 (dmat->alignment < dmat->maxsize) && 578 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 579 free(vaddr, M_DEVBUF); 580 else { 581 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 582 } 583 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 584} 585 586int 587_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 588 void *buf, bus_size_t buflen, int flags) 589{ 590 vm_offset_t vaddr; 591 vm_offset_t vendaddr; 592 bus_addr_t paddr; 593 594 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { 595 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 596 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), 597 dmat->boundary, dmat->alignment); 598 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", 599 map, &nobounce_dmamap, map->pagesneeded); 600 /* 601 * Count the number of bounce pages 602 * needed in order to complete this transfer 603 */ 604 vaddr = (vm_offset_t)buf; 605 vendaddr = (vm_offset_t)buf + buflen; 606 607 while (vaddr < vendaddr) { 608 bus_size_t sg_len; 609 610 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 611 if (pmap) 612 paddr = pmap_extract(pmap, vaddr); 613 else 614 paddr = pmap_kextract(vaddr); 615 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 616 run_filter(dmat, paddr) != 0) { 617 sg_len = roundup2(sg_len, dmat->alignment); 618 map->pagesneeded++; 619 } 620 vaddr += sg_len; 621 } 622 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 623 } 624 625 /* Reserve Necessary Bounce Pages */ 626 if (map->pagesneeded != 0) { 627 mtx_lock(&bounce_lock); 628 if (flags & BUS_DMA_NOWAIT) { 629 if (reserve_bounce_pages(dmat, map, 0) != 0) { 630 mtx_unlock(&bounce_lock); 631 return (ENOMEM); 632 } 633 } else { 634 if (reserve_bounce_pages(dmat, map, 1) != 0) { 635 /* Queue us for resources */ 636 map->dmat = dmat; 637 map->buf = buf; 638 map->buflen = buflen; 639 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 640 map, links); 641 mtx_unlock(&bounce_lock); 642 return (EINPROGRESS); 643 } 644 } 645 mtx_unlock(&bounce_lock); 646 } 647 648 return (0); 649} 650 651/* 652 * Utility function to load a linear buffer. lastaddrp holds state 653 * between invocations (for multiple-buffer loads). segp contains 654 * the starting segment on entrace, and the ending segment on exit. 655 * first indicates if this is the first invocation of this function. 656 */ 657static __inline int 658_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 659 bus_dmamap_t map, 660 void *buf, bus_size_t buflen, 661 pmap_t pmap, 662 int flags, 663 bus_addr_t *lastaddrp, 664 bus_dma_segment_t *segs, 665 int *segp, 666 int first) 667{ 668 bus_size_t sgsize; 669 bus_addr_t curaddr, lastaddr, baddr, bmask; 670 vm_offset_t vaddr; 671 int seg, error; 672 673 if (map == NULL) 674 map = &nobounce_dmamap; 675 676 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 677 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 678 if (error) 679 return (error); 680 } 681 682 vaddr = (vm_offset_t)buf; 683 lastaddr = *lastaddrp; 684 bmask = ~(dmat->boundary - 1); 685 686 for (seg = *segp; buflen > 0 ; ) { 687 bus_size_t max_sgsize; 688 689 /* 690 * Get the physical address for this segment. 691 */ 692 if (pmap) 693 curaddr = pmap_extract(pmap, vaddr); 694 else 695 curaddr = pmap_kextract(vaddr); 696 697 /* 698 * Compute the segment size, and adjust counts. 699 */ 700 max_sgsize = MIN(buflen, dmat->maxsegsz); 701 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK); 702 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 703 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 704 sgsize = roundup2(sgsize, dmat->alignment); 705 sgsize = MIN(sgsize, max_sgsize); 706 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 707 } else { 708 sgsize = MIN(sgsize, max_sgsize); 709 } 710 711 /* 712 * Make sure we don't cross any boundaries. 713 */ 714 if (dmat->boundary > 0) { 715 baddr = (curaddr + dmat->boundary) & bmask; 716 if (sgsize > (baddr - curaddr)) 717 sgsize = (baddr - curaddr); 718 } 719 720 /* 721 * Insert chunk into a segment, coalescing with 722 * previous segment if possible. 723 */ 724 if (first) { 725 segs[seg].ds_addr = curaddr; 726 segs[seg].ds_len = sgsize; 727 first = 0; 728 } else { 729 if (curaddr == lastaddr && 730 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 731 (dmat->boundary == 0 || 732 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 733 segs[seg].ds_len += sgsize; 734 else { 735 if (++seg >= dmat->nsegments) 736 break; 737 segs[seg].ds_addr = curaddr; 738 segs[seg].ds_len = sgsize; 739 } 740 } 741 742 lastaddr = curaddr + sgsize; 743 vaddr += sgsize; 744 buflen -= sgsize; 745 } 746 747 *segp = seg; 748 *lastaddrp = lastaddr; 749 750 /* 751 * Did we fit? 752 */ 753 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 754} 755 756/* 757 * Map the buffer buf into bus space using the dmamap map. 758 */ 759int 760bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 761 bus_size_t buflen, bus_dmamap_callback_t *callback, 762 void *callback_arg, int flags) 763{ 764 bus_addr_t lastaddr = 0; 765 int error, nsegs = 0; 766 767 if (map != NULL) { 768 flags |= BUS_DMA_WAITOK; 769 map->callback = callback; 770 map->callback_arg = callback_arg; 771 } 772 773 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 774 &lastaddr, dmat->segments, &nsegs, 1); 775 776 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 777 __func__, dmat, dmat->flags, error, nsegs + 1); 778 779 if (error == EINPROGRESS) { 780 return (error); 781 } 782 783 if (error) 784 (*callback)(callback_arg, dmat->segments, 0, error); 785 else 786 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 787 788 /* 789 * Return ENOMEM to the caller so that it can pass it up the stack. 790 * This error only happens when NOWAIT is set, so deferal is disabled. 791 */ 792 if (error == ENOMEM) 793 return (error); 794 795 return (0); 796} 797 798 799/* 800 * Like _bus_dmamap_load(), but for mbufs. 801 */ 802static __inline int 803_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 804 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 805 int flags) 806{ 807 int error; 808 809 M_ASSERTPKTHDR(m0); 810 811 flags |= BUS_DMA_NOWAIT; 812 *nsegs = 0; 813 error = 0; 814 if (m0->m_pkthdr.len <= dmat->maxsize) { 815 int first = 1; 816 bus_addr_t lastaddr = 0; 817 struct mbuf *m; 818 819 for (m = m0; m != NULL && error == 0; m = m->m_next) { 820 if (m->m_len > 0) { 821 error = _bus_dmamap_load_buffer(dmat, map, 822 m->m_data, m->m_len, 823 NULL, flags, &lastaddr, 824 segs, nsegs, first); 825 first = 0; 826 } 827 } 828 } else { 829 error = EINVAL; 830 } 831 832 /* XXX FIXME: Having to increment nsegs is really annoying */ 833 ++*nsegs; 834 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 835 __func__, dmat, dmat->flags, error, *nsegs); 836 return (error); 837} 838 839int 840bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 841 struct mbuf *m0, 842 bus_dmamap_callback2_t *callback, void *callback_arg, 843 int flags) 844{ 845 int nsegs, error; 846 847 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, 848 flags); 849 850 if (error) { 851 /* force "no valid mappings" in callback */ 852 (*callback)(callback_arg, dmat->segments, 0, 0, error); 853 } else { 854 (*callback)(callback_arg, dmat->segments, 855 nsegs, m0->m_pkthdr.len, error); 856 } 857 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 858 __func__, dmat, dmat->flags, error, nsegs); 859 return (error); 860} 861 862int 863bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 864 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 865 int flags) 866{ 867 return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); 868} 869 870/* 871 * Like _bus_dmamap_load(), but for uios. 872 */ 873int 874bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 875 struct uio *uio, 876 bus_dmamap_callback2_t *callback, void *callback_arg, 877 int flags) 878{ 879 bus_addr_t lastaddr = 0; 880 int nsegs, error, first, i; 881 bus_size_t resid; 882 struct iovec *iov; 883 pmap_t pmap; 884 885 flags |= BUS_DMA_NOWAIT; 886 resid = uio->uio_resid; 887 iov = uio->uio_iov; 888 889 if (uio->uio_segflg == UIO_USERSPACE) { 890 KASSERT(uio->uio_td != NULL, 891 ("bus_dmamap_load_uio: USERSPACE but no proc")); 892 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 893 } else 894 pmap = NULL; 895 896 nsegs = 0; 897 error = 0; 898 first = 1; 899 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 900 /* 901 * Now at the first iovec to load. Load each iovec 902 * until we have exhausted the residual count. 903 */ 904 bus_size_t minlen = 905 resid < iov[i].iov_len ? resid : iov[i].iov_len; 906 caddr_t addr = (caddr_t) iov[i].iov_base; 907 908 if (minlen > 0) { 909 error = _bus_dmamap_load_buffer(dmat, map, 910 addr, minlen, pmap, flags, &lastaddr, 911 dmat->segments, &nsegs, first); 912 first = 0; 913 914 resid -= minlen; 915 } 916 } 917 918 if (error) { 919 /* force "no valid mappings" in callback */ 920 (*callback)(callback_arg, dmat->segments, 0, 0, error); 921 } else { 922 (*callback)(callback_arg, dmat->segments, 923 nsegs+1, uio->uio_resid, error); 924 } 925 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 926 __func__, dmat, dmat->flags, error, nsegs + 1); 927 return (error); 928} 929 930/* 931 * Release the mapping held by map. 932 */ 933void 934_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 935{ 936 struct bounce_page *bpage; 937 938 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 939 STAILQ_REMOVE_HEAD(&map->bpages, links); 940 free_bounce_page(dmat, bpage); 941 } 942} 943 944void 945_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 946{ 947 struct bounce_page *bpage; 948 949 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 950 /* 951 * Handle data bouncing. We might also 952 * want to add support for invalidating 953 * the caches on broken hardware 954 */ 955 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 956 "performing bounce", __func__, op, dmat, dmat->flags); 957 958 if (op & BUS_DMASYNC_PREWRITE) { 959 while (bpage != NULL) { 960 bcopy((void *)bpage->datavaddr, 961 (void *)bpage->vaddr, 962 bpage->datacount); 963 bpage = STAILQ_NEXT(bpage, links); 964 } 965 dmat->bounce_zone->total_bounced++; 966 } 967 968 if (op & BUS_DMASYNC_POSTREAD) { 969 while (bpage != NULL) { 970 bcopy((void *)bpage->vaddr, 971 (void *)bpage->datavaddr, 972 bpage->datacount); 973 bpage = STAILQ_NEXT(bpage, links); 974 } 975 dmat->bounce_zone->total_bounced++; 976 } 977 } 978} 979 980static void 981init_bounce_pages(void *dummy __unused) 982{ 983 984 total_bpages = 0; 985 STAILQ_INIT(&bounce_zone_list); 986 STAILQ_INIT(&bounce_map_waitinglist); 987 STAILQ_INIT(&bounce_map_callbacklist); 988 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 989} 990SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 991 992static struct sysctl_ctx_list * 993busdma_sysctl_tree(struct bounce_zone *bz) 994{ 995 return (&bz->sysctl_tree); 996} 997 998static struct sysctl_oid * 999busdma_sysctl_tree_top(struct bounce_zone *bz) 1000{ 1001 return (bz->sysctl_tree_top); 1002} 1003 1004#if defined(__amd64__) || defined(PAE) 1005#define SYSCTL_ADD_BUS_SIZE_T SYSCTL_ADD_UQUAD 1006#else 1007#define SYSCTL_ADD_BUS_SIZE_T(ctx, parent, nbr, name, flag, ptr, desc) \ 1008 SYSCTL_ADD_UINT(ctx, parent, nbr, name, flag, ptr, 0, desc) 1009#endif 1010 1011static int 1012alloc_bounce_zone(bus_dma_tag_t dmat) 1013{ 1014 struct bounce_zone *bz; 1015 1016 /* Check to see if we already have a suitable zone */ 1017 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1018 if ((dmat->alignment <= bz->alignment) 1019 && (dmat->lowaddr >= bz->lowaddr)) { 1020 dmat->bounce_zone = bz; 1021 return (0); 1022 } 1023 } 1024 1025 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1026 M_NOWAIT | M_ZERO)) == NULL) 1027 return (ENOMEM); 1028 1029 STAILQ_INIT(&bz->bounce_page_list); 1030 bz->free_bpages = 0; 1031 bz->reserved_bpages = 0; 1032 bz->active_bpages = 0; 1033 bz->lowaddr = dmat->lowaddr; 1034 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1035 bz->map_count = 0; 1036 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1037 busdma_zonecount++; 1038 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1039 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1040 dmat->bounce_zone = bz; 1041 1042 sysctl_ctx_init(&bz->sysctl_tree); 1043 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1044 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1045 CTLFLAG_RD, 0, ""); 1046 if (bz->sysctl_tree_top == NULL) { 1047 sysctl_ctx_free(&bz->sysctl_tree); 1048 return (0); /* XXX error code? */ 1049 } 1050 1051 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1052 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1053 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1054 "Total bounce pages"); 1055 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1056 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1057 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1058 "Free bounce pages"); 1059 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1060 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1061 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1062 "Reserved bounce pages"); 1063 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1064 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1065 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1066 "Active bounce pages"); 1067 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1068 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1069 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1070 "Total bounce requests"); 1071 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1072 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1073 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1074 "Total bounce requests that were deferred"); 1075 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1076 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1077 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1078 SYSCTL_ADD_BUS_SIZE_T(busdma_sysctl_tree(bz), 1079 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1080 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1081 1082 return (0); 1083} 1084 1085static int 1086alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1087{ 1088 struct bounce_zone *bz; 1089 int count; 1090 1091 bz = dmat->bounce_zone; 1092 count = 0; 1093 while (numpages > 0) { 1094 struct bounce_page *bpage; 1095 1096 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1097 M_NOWAIT | M_ZERO); 1098 1099 if (bpage == NULL) 1100 break; 1101 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1102 M_NOWAIT, 0ul, 1103 bz->lowaddr, 1104 PAGE_SIZE, 1105 0); 1106 if (bpage->vaddr == 0) { 1107 free(bpage, M_DEVBUF); 1108 break; 1109 } 1110 bpage->busaddr = pmap_kextract(bpage->vaddr); 1111 mtx_lock(&bounce_lock); 1112 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1113 total_bpages++; 1114 bz->total_bpages++; 1115 bz->free_bpages++; 1116 mtx_unlock(&bounce_lock); 1117 count++; 1118 numpages--; 1119 } 1120 return (count); 1121} 1122 1123static int 1124reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1125{ 1126 struct bounce_zone *bz; 1127 int pages; 1128 1129 mtx_assert(&bounce_lock, MA_OWNED); 1130 bz = dmat->bounce_zone; 1131 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1132 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1133 return (map->pagesneeded - (map->pagesreserved + pages)); 1134 bz->free_bpages -= pages; 1135 bz->reserved_bpages += pages; 1136 map->pagesreserved += pages; 1137 pages = map->pagesneeded - map->pagesreserved; 1138 1139 return (pages); 1140} 1141 1142static bus_addr_t 1143add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1144 bus_size_t size) 1145{ 1146 struct bounce_zone *bz; 1147 struct bounce_page *bpage; 1148 1149 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1150 KASSERT(map != NULL && map != &nobounce_dmamap, 1151 ("add_bounce_page: bad map %p", map)); 1152 1153 bz = dmat->bounce_zone; 1154 if (map->pagesneeded == 0) 1155 panic("add_bounce_page: map doesn't need any pages"); 1156 map->pagesneeded--; 1157 1158 if (map->pagesreserved == 0) 1159 panic("add_bounce_page: map doesn't need any pages"); 1160 map->pagesreserved--; 1161 1162 mtx_lock(&bounce_lock); 1163 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1164 if (bpage == NULL) 1165 panic("add_bounce_page: free page list is empty"); 1166 1167 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1168 bz->reserved_bpages--; 1169 bz->active_bpages++; 1170 mtx_unlock(&bounce_lock); 1171 1172 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1173 /* Page offset needs to be preserved. */ 1174 bpage->vaddr |= vaddr & PAGE_MASK; 1175 bpage->busaddr |= vaddr & PAGE_MASK; 1176 } 1177 bpage->datavaddr = vaddr; 1178 bpage->datacount = size; 1179 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1180 return (bpage->busaddr); 1181} 1182 1183static void 1184free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1185{ 1186 struct bus_dmamap *map; 1187 struct bounce_zone *bz; 1188 1189 bz = dmat->bounce_zone; 1190 bpage->datavaddr = 0; 1191 bpage->datacount = 0; 1192 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1193 /* 1194 * Reset the bounce page to start at offset 0. Other uses 1195 * of this bounce page may need to store a full page of 1196 * data and/or assume it starts on a page boundary. 1197 */ 1198 bpage->vaddr &= ~PAGE_MASK; 1199 bpage->busaddr &= ~PAGE_MASK; 1200 } 1201 1202 mtx_lock(&bounce_lock); 1203 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1204 bz->free_bpages++; 1205 bz->active_bpages--; 1206 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1207 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1208 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1209 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1210 map, links); 1211 busdma_swi_pending = 1; 1212 bz->total_deferred++; 1213 swi_sched(vm_ih, 0); 1214 } 1215 } 1216 mtx_unlock(&bounce_lock); 1217} 1218 1219void 1220busdma_swi(void) 1221{ 1222 bus_dma_tag_t dmat; 1223 struct bus_dmamap *map; 1224 1225 mtx_lock(&bounce_lock); 1226 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1227 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1228 mtx_unlock(&bounce_lock); 1229 dmat = map->dmat; 1230 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1231 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1232 map->callback, map->callback_arg, /*flags*/0); 1233 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1234 mtx_lock(&bounce_lock); 1235 } 1236 mtx_unlock(&bounce_lock); 1237} 1238