busdma_machdep-v4.c revision 289851
1169689Skan/*- 2169689Skan * Copyright (c) 2012 Ian Lepore 3169689Skan * Copyright (c) 2004 Olivier Houchard 4169689Skan * Copyright (c) 2002 Peter Grehan 5169689Skan * Copyright (c) 1997, 1998 Justin T. Gibbs. 6169689Skan * All rights reserved. 7169689Skan * 8169689Skan * Redistribution and use in source and binary forms, with or without 9169689Skan * modification, are permitted provided that the following conditions 10169689Skan * are met: 11169689Skan * 1. Redistributions of source code must retain the above copyright 12169689Skan * notice, this list of conditions, and the following disclaimer, 13169689Skan * without modification, immediately at the beginning of the file. 14169689Skan * 2. The name of the author may not be used to endorse or promote products 15169689Skan * derived from this software without specific prior written permission. 16169689Skan * 17169689Skan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18169689Skan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19169689Skan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20169689Skan * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21169689Skan * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22169689Skan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23169689Skan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24169689Skan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25169689Skan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26169689Skan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27169689Skan * SUCH DAMAGE. 28169689Skan * 29169689Skan * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30169689Skan */ 31169689Skan 32169689Skan#include <sys/cdefs.h> 33169689Skan__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 289851 2015-10-23 20:49:34Z ian $"); 34169689Skan 35169689Skan/* 36169689Skan * ARM bus dma support routines. 37169689Skan * 38169689Skan * XXX Things to investigate / fix some day... 39169689Skan * - What is the earliest that this API can be called? Could there be any 40169689Skan * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41169689Skan * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42169689Skan * bus_dmamap_load() function. This code has historically (and still does) 43169689Skan * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44169689Skan * error checking because some resource management calls would become WAITOK 45169689Skan * and thus "cannot fail." 46169689Skan * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47169689Skan * creation time, and the result stored in the tag. 48169689Skan * - It should be possible to take some shortcuts when mapping a buffer we know 49169689Skan * came from the uma(9) allocators based on what we know about such buffers 50169689Skan * (aligned, contiguous, etc). 51169689Skan * - The allocation of bounce pages could probably be cleaned up, then we could 52169689Skan * retire arm_remap_nocache(). 53169689Skan */ 54169689Skan 55169689Skan#define _ARM32_BUS_DMA_PRIVATE 56169689Skan#include <sys/param.h> 57169689Skan#include <sys/systm.h> 58169689Skan#include <sys/malloc.h> 59169689Skan#include <sys/bus.h> 60169689Skan#include <sys/busdma_bufalloc.h> 61169689Skan#include <sys/interrupt.h> 62169689Skan#include <sys/lock.h> 63169689Skan#include <sys/proc.h> 64169689Skan#include <sys/memdesc.h> 65169689Skan#include <sys/mutex.h> 66169689Skan#include <sys/ktr.h> 67169689Skan#include <sys/kernel.h> 68169689Skan#include <sys/sysctl.h> 69169689Skan#include <sys/uio.h> 70169689Skan 71169689Skan#include <vm/uma.h> 72169689Skan#include <vm/vm.h> 73169689Skan#include <vm/vm_extern.h> 74169689Skan#include <vm/vm_kern.h> 75169689Skan#include <vm/vm_page.h> 76169689Skan#include <vm/vm_map.h> 77169689Skan 78169689Skan#include <machine/atomic.h> 79169689Skan#include <machine/bus.h> 80169689Skan#include <machine/cpufunc.h> 81169689Skan#include <machine/md_var.h> 82169689Skan 83169689Skan#define MAX_BPAGES 64 84169689Skan#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85169689Skan#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86169689Skan 87169689Skanstruct bounce_zone; 88169689Skan 89169689Skanstruct bus_dma_tag { 90169689Skan bus_dma_tag_t parent; 91169689Skan bus_size_t alignment; 92169689Skan bus_addr_t boundary; 93169689Skan bus_addr_t lowaddr; 94169689Skan bus_addr_t highaddr; 95169689Skan bus_dma_filter_t *filter; 96169689Skan void *filterarg; 97169689Skan bus_size_t maxsize; 98169689Skan u_int nsegments; 99169689Skan bus_size_t maxsegsz; 100169689Skan int flags; 101169689Skan int ref_count; 102169689Skan int map_count; 103169689Skan bus_dma_lock_t *lockfunc; 104169689Skan void *lockfuncarg; 105169689Skan struct bounce_zone *bounce_zone; 106169689Skan /* 107169689Skan * DMA range for this tag. If the page doesn't fall within 108169689Skan * one of these ranges, an error is returned. The caller 109169689Skan * may then decide what to do with the transfer. If the 110169689Skan * range pointer is NULL, it is ignored. 111169689Skan */ 112169689Skan struct arm32_dma_range *ranges; 113169689Skan int _nranges; 114169689Skan /* 115169689Skan * Most tags need one or two segments, and can use the local tagsegs 116169689Skan * array. For tags with a larger limit, we'll allocate a bigger array 117169689Skan * on first use. 118169689Skan */ 119169689Skan bus_dma_segment_t *segments; 120169689Skan bus_dma_segment_t tagsegs[2]; 121169689Skan}; 122169689Skan 123169689Skanstruct bounce_page { 124169689Skan vm_offset_t vaddr; /* kva of bounce buffer */ 125169689Skan bus_addr_t busaddr; /* Physical address */ 126169689Skan vm_offset_t datavaddr; /* kva of client data */ 127169689Skan vm_page_t datapage; /* physical page of client data */ 128169689Skan vm_offset_t dataoffs; /* page offset of client data */ 129169689Skan bus_size_t datacount; /* client data count */ 130169689Skan STAILQ_ENTRY(bounce_page) links; 131169689Skan}; 132169689Skan 133169689Skanstruct sync_list { 134169689Skan vm_offset_t vaddr; /* kva of client data */ 135169689Skan vm_page_t pages; /* starting page of client data */ 136169689Skan vm_offset_t dataoffs; /* page offset of client data */ 137169689Skan bus_size_t datacount; /* client data count */ 138169689Skan}; 139169689Skan 140169689Skanint busdma_swi_pending; 141169689Skan 142169689Skanstruct bounce_zone { 143169689Skan STAILQ_ENTRY(bounce_zone) links; 144169689Skan STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 145169689Skan int total_bpages; 146169689Skan int free_bpages; 147169689Skan int reserved_bpages; 148169689Skan int active_bpages; 149169689Skan int total_bounced; 150169689Skan int total_deferred; 151169689Skan int map_count; 152169689Skan bus_size_t alignment; 153169689Skan bus_addr_t lowaddr; 154169689Skan char zoneid[8]; 155169689Skan char lowaddrid[20]; 156169689Skan struct sysctl_ctx_list sysctl_tree; 157169689Skan struct sysctl_oid *sysctl_tree_top; 158169689Skan}; 159169689Skan 160169689Skanstatic struct mtx bounce_lock; 161169689Skanstatic int total_bpages; 162169689Skanstatic int busdma_zonecount; 163static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 164 165static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 166SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 167 "Total bounce pages"); 168 169struct bus_dmamap { 170 struct bp_list bpages; 171 int pagesneeded; 172 int pagesreserved; 173 bus_dma_tag_t dmat; 174 struct memdesc mem; 175 bus_dmamap_callback_t *callback; 176 void *callback_arg; 177 int flags; 178#define DMAMAP_COHERENT 0x8 179#define DMAMAP_CACHE_ALIGNED 0x10 180 STAILQ_ENTRY(bus_dmamap) links; 181 int sync_count; 182 struct sync_list *slist; 183}; 184 185static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 186static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 187 188static struct mtx busdma_mtx; 189 190MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 191 192static void init_bounce_pages(void *dummy); 193static int alloc_bounce_zone(bus_dma_tag_t dmat); 194static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 195static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 196 int commit); 197static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 198 vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 199static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 200static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 201 int bufaligned); 202 203/* Default tag, as most drivers provide no parent tag. */ 204bus_dma_tag_t arm_root_dma_tag; 205 206/* 207 * ---------------------------------------------------------------------------- 208 * Begin block of code useful to transplant to other implementations. 209 */ 210 211static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 212 213static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 214static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 215 216/* 217 * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 218 * It'll need platform-specific changes if this code is copied. 219 */ 220static int 221dmamap_ctor(void *mem, int size, void *arg, int flags) 222{ 223 bus_dmamap_t map; 224 bus_dma_tag_t dmat; 225 226 map = (bus_dmamap_t)mem; 227 dmat = (bus_dma_tag_t)arg; 228 229 dmat->map_count++; 230 231 map->dmat = dmat; 232 map->flags = 0; 233 STAILQ_INIT(&map->bpages); 234 235 return (0); 236} 237 238/* 239 * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 240 * It may need platform-specific changes if this code is copied . 241 */ 242static void 243dmamap_dtor(void *mem, int size, void *arg) 244{ 245 bus_dmamap_t map; 246 247 map = (bus_dmamap_t)mem; 248 249 map->dmat->map_count--; 250} 251 252static void 253busdma_init(void *dummy) 254{ 255 256 /* Create a cache of maps for bus_dmamap_create(). */ 257 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 258 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 259 260 /* Create a cache of buffers in standard (cacheable) memory. */ 261 standard_allocator = busdma_bufalloc_create("buffer", 262 arm_dcache_align, /* minimum_alignment */ 263 NULL, /* uma_alloc func */ 264 NULL, /* uma_free func */ 265 0); /* uma_zcreate_flags */ 266 267 /* 268 * Create a cache of buffers in uncacheable memory, to implement the 269 * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 270 */ 271 coherent_allocator = busdma_bufalloc_create("coherent", 272 arm_dcache_align, /* minimum_alignment */ 273 busdma_bufalloc_alloc_uncacheable, 274 busdma_bufalloc_free_uncacheable, 275 0); /* uma_zcreate_flags */ 276} 277 278/* 279 * This init historically used SI_SUB_VM, but now the init code requires 280 * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 281 * SI_SUB_KMEM and SI_ORDER_THIRD, so we'll go right after that by using 282 * SI_SUB_KMEM and SI_ORDER_FOURTH. 283 */ 284SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL); 285 286/* 287 * End block of code useful to transplant to other implementations. 288 * ---------------------------------------------------------------------------- 289 */ 290 291/* 292 * Return true if a match is made. 293 * 294 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 295 * 296 * If paddr is within the bounds of the dma tag then call the filter callback 297 * to check for a match, if there is no filter callback then assume a match. 298 */ 299static int 300run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 301{ 302 int retval; 303 304 retval = 0; 305 306 do { 307 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 308 || ((paddr & (dmat->alignment - 1)) != 0)) 309 && (dmat->filter == NULL 310 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 311 retval = 1; 312 313 dmat = dmat->parent; 314 } while (retval == 0 && dmat != NULL); 315 return (retval); 316} 317 318/* 319 * This routine checks the exclusion zone constraints from a tag against the 320 * physical RAM available on the machine. If a tag specifies an exclusion zone 321 * but there's no RAM in that zone, then we avoid allocating resources to bounce 322 * a request, and we can use any memory allocator (as opposed to needing 323 * kmem_alloc_contig() just because it can allocate pages in an address range). 324 * 325 * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 326 * same value on 32-bit architectures) as their lowaddr constraint, and we can't 327 * possibly have RAM at an address higher than the highest address we can 328 * express, so we take a fast out. 329 */ 330static __inline int 331_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 332{ 333 int i; 334 335 if (lowaddr >= BUS_SPACE_MAXADDR) 336 return (0); 337 338 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 339 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 340 || (lowaddr < phys_avail[i] && 341 highaddr > phys_avail[i])) 342 return (1); 343 } 344 return (0); 345} 346 347static __inline struct arm32_dma_range * 348_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 349 bus_addr_t curaddr) 350{ 351 struct arm32_dma_range *dr; 352 int i; 353 354 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 355 if (curaddr >= dr->dr_sysbase && 356 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 357 return (dr); 358 } 359 360 return (NULL); 361} 362 363/* 364 * Convenience function for manipulating driver locks from busdma (during 365 * busdma_swi, for example). Drivers that don't provide their own locks 366 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 367 * non-mutex locking scheme don't have to use this at all. 368 */ 369void 370busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 371{ 372 struct mtx *dmtx; 373 374 dmtx = (struct mtx *)arg; 375 switch (op) { 376 case BUS_DMA_LOCK: 377 mtx_lock(dmtx); 378 break; 379 case BUS_DMA_UNLOCK: 380 mtx_unlock(dmtx); 381 break; 382 default: 383 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 384 } 385} 386 387/* 388 * dflt_lock should never get called. It gets put into the dma tag when 389 * lockfunc == NULL, which is only valid if the maps that are associated 390 * with the tag are meant to never be defered. 391 * XXX Should have a way to identify which driver is responsible here. 392 */ 393static void 394dflt_lock(void *arg, bus_dma_lock_op_t op) 395{ 396#ifdef INVARIANTS 397 panic("driver error: busdma dflt_lock called"); 398#else 399 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 400#endif 401} 402 403/* 404 * Allocate a device specific dma_tag. 405 */ 406#define SEG_NB 1024 407 408int 409bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 410 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 411 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 412 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 413 void *lockfuncarg, bus_dma_tag_t *dmat) 414{ 415 bus_dma_tag_t newtag; 416 int error = 0; 417 /* Return a NULL tag on failure */ 418 *dmat = NULL; 419 if (!parent) 420 parent = arm_root_dma_tag; 421 422 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 423 if (newtag == NULL) { 424 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 425 __func__, newtag, 0, error); 426 return (ENOMEM); 427 } 428 429 newtag->parent = parent; 430 newtag->alignment = alignment ? alignment : 1; 431 newtag->boundary = boundary; 432 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 433 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 434 newtag->filter = filter; 435 newtag->filterarg = filterarg; 436 newtag->maxsize = maxsize; 437 newtag->nsegments = nsegments; 438 newtag->maxsegsz = maxsegsz; 439 newtag->flags = flags; 440 newtag->ref_count = 1; /* Count ourself */ 441 newtag->map_count = 0; 442 newtag->ranges = bus_dma_get_range(); 443 newtag->_nranges = bus_dma_get_range_nb(); 444 if (lockfunc != NULL) { 445 newtag->lockfunc = lockfunc; 446 newtag->lockfuncarg = lockfuncarg; 447 } else { 448 newtag->lockfunc = dflt_lock; 449 newtag->lockfuncarg = NULL; 450 } 451 /* 452 * If all the segments we need fit into the local tagsegs array, set the 453 * pointer now. Otherwise NULL the pointer and an array of segments 454 * will be allocated later, on first use. We don't pre-allocate now 455 * because some tags exist just to pass contraints to children in the 456 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 457 * sure don't want to try to allocate an array for that. 458 */ 459 if (newtag->nsegments <= nitems(newtag->tagsegs)) 460 newtag->segments = newtag->tagsegs; 461 else 462 newtag->segments = NULL; 463 /* 464 * Take into account any restrictions imposed by our parent tag 465 */ 466 if (parent != NULL) { 467 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 468 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 469 if (newtag->boundary == 0) 470 newtag->boundary = parent->boundary; 471 else if (parent->boundary != 0) 472 newtag->boundary = MIN(parent->boundary, 473 newtag->boundary); 474 if ((newtag->filter != NULL) || 475 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 476 newtag->flags |= BUS_DMA_COULD_BOUNCE; 477 if (newtag->filter == NULL) { 478 /* 479 * Short circuit looking at our parent directly 480 * since we have encapsulated all of its information 481 */ 482 newtag->filter = parent->filter; 483 newtag->filterarg = parent->filterarg; 484 newtag->parent = parent->parent; 485 } 486 if (newtag->parent != NULL) 487 atomic_add_int(&parent->ref_count, 1); 488 } 489 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 490 || newtag->alignment > 1) 491 newtag->flags |= BUS_DMA_COULD_BOUNCE; 492 493 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 494 (flags & BUS_DMA_ALLOCNOW) != 0) { 495 struct bounce_zone *bz; 496 497 /* Must bounce */ 498 499 if ((error = alloc_bounce_zone(newtag)) != 0) { 500 free(newtag, M_DEVBUF); 501 return (error); 502 } 503 bz = newtag->bounce_zone; 504 505 if (ptoa(bz->total_bpages) < maxsize) { 506 int pages; 507 508 pages = atop(maxsize) - bz->total_bpages; 509 510 /* Add pages to our bounce pool */ 511 if (alloc_bounce_pages(newtag, pages) < pages) 512 error = ENOMEM; 513 } 514 /* Performed initial allocation */ 515 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 516 } else 517 newtag->bounce_zone = NULL; 518 if (error != 0) 519 free(newtag, M_DEVBUF); 520 else 521 *dmat = newtag; 522 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 523 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 524 525 return (error); 526} 527 528int 529bus_dma_tag_destroy(bus_dma_tag_t dmat) 530{ 531#ifdef KTR 532 bus_dma_tag_t dmat_copy = dmat; 533#endif 534 535 if (dmat != NULL) { 536 537 if (dmat->map_count != 0) 538 return (EBUSY); 539 540 while (dmat != NULL) { 541 bus_dma_tag_t parent; 542 543 parent = dmat->parent; 544 atomic_subtract_int(&dmat->ref_count, 1); 545 if (dmat->ref_count == 0) { 546 if (dmat->segments != NULL && 547 dmat->segments != dmat->tagsegs) 548 free(dmat->segments, M_DEVBUF); 549 free(dmat, M_DEVBUF); 550 /* 551 * Last reference count, so 552 * release our reference 553 * count on our parent. 554 */ 555 dmat = parent; 556 } else 557 dmat = NULL; 558 } 559 } 560 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 561 562 return (0); 563} 564 565#include <sys/kdb.h> 566/* 567 * Allocate a handle for mapping from kva/uva/physical 568 * address space into bus device space. 569 */ 570int 571bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 572{ 573 struct sync_list *slist; 574 bus_dmamap_t map; 575 int error = 0; 576 577 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 578 if (slist == NULL) 579 return (ENOMEM); 580 581 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); 582 *mapp = map; 583 if (map == NULL) { 584 free(slist, M_DEVBUF); 585 return (ENOMEM); 586 } 587 588 /* 589 * If the tag's segments haven't been allocated yet we need to do it 590 * now, because we can't sleep for resources at map load time. 591 */ 592 if (dmat->segments == NULL) { 593 dmat->segments = malloc(dmat->nsegments * 594 sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); 595 if (dmat->segments == NULL) { 596 free(slist, M_DEVBUF); 597 uma_zfree(dmamap_zone, map); 598 *mapp = NULL; 599 return (ENOMEM); 600 } 601 } 602 603 /* 604 * Bouncing might be required if the driver asks for an active 605 * exclusion region, a data alignment that is stricter than 1, and/or 606 * an active address boundary. 607 */ 608 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 609 610 /* Must bounce */ 611 struct bounce_zone *bz; 612 int maxpages; 613 614 if (dmat->bounce_zone == NULL) { 615 if ((error = alloc_bounce_zone(dmat)) != 0) { 616 free(slist, M_DEVBUF); 617 uma_zfree(dmamap_zone, map); 618 *mapp = NULL; 619 return (error); 620 } 621 } 622 bz = dmat->bounce_zone; 623 624 /* Initialize the new map */ 625 STAILQ_INIT(&((*mapp)->bpages)); 626 627 /* 628 * Attempt to add pages to our pool on a per-instance 629 * basis up to a sane limit. 630 */ 631 maxpages = MAX_BPAGES; 632 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 633 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 634 int pages; 635 636 pages = MAX(atop(dmat->maxsize), 1); 637 pages = MIN(maxpages - bz->total_bpages, pages); 638 pages = MAX(pages, 1); 639 if (alloc_bounce_pages(dmat, pages) < pages) 640 error = ENOMEM; 641 642 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 643 if (error == 0) 644 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 645 } else { 646 error = 0; 647 } 648 } 649 bz->map_count++; 650 } 651 map->sync_count = 0; 652 map->slist = slist; 653 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 654 __func__, dmat, dmat->flags, error); 655 656 return (0); 657} 658 659/* 660 * Destroy a handle for mapping from kva/uva/physical 661 * address space into bus device space. 662 */ 663int 664bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 665{ 666 667 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 668 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 669 __func__, dmat, EBUSY); 670 return (EBUSY); 671 } 672 free(map->slist, M_DEVBUF); 673 uma_zfree(dmamap_zone, map); 674 if (dmat->bounce_zone) 675 dmat->bounce_zone->map_count--; 676 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 677 return (0); 678} 679 680/* 681 * Allocate a piece of memory that can be efficiently mapped into bus device 682 * space based on the constraints listed in the dma tag. Returns a pointer to 683 * the allocated memory, and a pointer to an associated bus_dmamap. 684 */ 685int 686bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 687 bus_dmamap_t *mapp) 688{ 689 struct sync_list *slist; 690 void * vaddr; 691 struct busdma_bufzone *bufzone; 692 busdma_bufalloc_t ba; 693 bus_dmamap_t map; 694 int mflags; 695 vm_memattr_t memattr; 696 697 if (flags & BUS_DMA_NOWAIT) 698 mflags = M_NOWAIT; 699 else 700 mflags = M_WAITOK; 701 /* 702 * If the tag's segments haven't been allocated yet we need to do it 703 * now, because we can't sleep for resources at map load time. 704 */ 705 if (dmat->segments == NULL) 706 dmat->segments = malloc(dmat->nsegments * 707 sizeof(*dmat->segments), M_DEVBUF, mflags); 708 709 slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); 710 if (slist == NULL) 711 return (ENOMEM); 712 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 713 if (map == NULL) { 714 free(slist, M_DEVBUF); 715 return (ENOMEM); 716 } 717 if (flags & BUS_DMA_COHERENT) { 718 memattr = VM_MEMATTR_UNCACHEABLE; 719 ba = coherent_allocator; 720 map->flags |= DMAMAP_COHERENT; 721 } else { 722 memattr = VM_MEMATTR_DEFAULT; 723 ba = standard_allocator; 724 } 725 /* All buffers we allocate are cache-aligned. */ 726 map->flags |= DMAMAP_CACHE_ALIGNED; 727 728 if (flags & BUS_DMA_ZERO) 729 mflags |= M_ZERO; 730 731 /* 732 * Try to find a bufzone in the allocator that holds a cache of buffers 733 * of the right size for this request. If the buffer is too big to be 734 * held in the allocator cache, this returns NULL. 735 */ 736 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 737 738 /* 739 * Allocate the buffer from the uma(9) allocator if... 740 * - It's small enough to be in the allocator (bufzone not NULL). 741 * - The alignment constraint isn't larger than the allocation size 742 * (the allocator aligns buffers to their size boundaries). 743 * - There's no need to handle lowaddr/highaddr exclusion zones. 744 * else allocate non-contiguous pages if... 745 * - The page count that could get allocated doesn't exceed nsegments. 746 * - The alignment constraint isn't larger than a page boundary. 747 * - There are no boundary-crossing constraints. 748 * else allocate a block of contiguous pages because one or more of the 749 * constraints is something that only the contig allocator can fulfill. 750 */ 751 if (bufzone != NULL && dmat->alignment <= bufzone->size && 752 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 753 vaddr = uma_zalloc(bufzone->umazone, mflags); 754 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 755 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 756 vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize, 757 mflags, 0, dmat->lowaddr, memattr); 758 } else { 759 vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize, 760 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 761 memattr); 762 } 763 if (vaddr == NULL) { 764 free(slist, M_DEVBUF); 765 uma_zfree(dmamap_zone, map); 766 map = NULL; 767 } else { 768 map->slist = slist; 769 map->sync_count = 0; 770 } 771 *vaddrp = vaddr; 772 *mapp = map; 773 774 return (vaddr == NULL ? ENOMEM : 0); 775} 776 777/* 778 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 779 * its associated map. 780 */ 781void 782bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 783{ 784 struct busdma_bufzone *bufzone; 785 busdma_bufalloc_t ba; 786 787 if (map->flags & DMAMAP_COHERENT) 788 ba = coherent_allocator; 789 else 790 ba = standard_allocator; 791 792 free(map->slist, M_DEVBUF); 793 uma_zfree(dmamap_zone, map); 794 795 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 796 797 if (bufzone != NULL && dmat->alignment <= bufzone->size && 798 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 799 uma_zfree(bufzone->umazone, vaddr); 800 else 801 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize); 802} 803 804static void 805_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 806 bus_size_t buflen, int flags) 807{ 808 bus_addr_t curaddr; 809 bus_size_t sgsize; 810 811 if (map->pagesneeded == 0) { 812 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 813 dmat->lowaddr, dmat->boundary, dmat->alignment); 814 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 815 map, map->pagesneeded); 816 /* 817 * Count the number of bounce pages 818 * needed in order to complete this transfer 819 */ 820 curaddr = buf; 821 while (buflen != 0) { 822 sgsize = MIN(buflen, dmat->maxsegsz); 823 if (run_filter(dmat, curaddr) != 0) { 824 sgsize = MIN(sgsize, 825 PAGE_SIZE - (curaddr & PAGE_MASK)); 826 map->pagesneeded++; 827 } 828 curaddr += sgsize; 829 buflen -= sgsize; 830 } 831 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 832 } 833} 834 835static void 836_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 837 void *buf, bus_size_t buflen, int flags) 838{ 839 vm_offset_t vaddr; 840 vm_offset_t vendaddr; 841 bus_addr_t paddr; 842 843 if (map->pagesneeded == 0) { 844 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 845 dmat->lowaddr, dmat->boundary, dmat->alignment); 846 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 847 map, map->pagesneeded); 848 /* 849 * Count the number of bounce pages 850 * needed in order to complete this transfer 851 */ 852 vaddr = trunc_page((vm_offset_t)buf); 853 vendaddr = (vm_offset_t)buf + buflen; 854 855 while (vaddr < vendaddr) { 856 if (__predict_true(pmap == kernel_pmap)) 857 paddr = pmap_kextract(vaddr); 858 else 859 paddr = pmap_extract(pmap, vaddr); 860 if (run_filter(dmat, paddr) != 0) 861 map->pagesneeded++; 862 vaddr += PAGE_SIZE; 863 } 864 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 865 } 866} 867 868static int 869_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 870{ 871 872 /* Reserve Necessary Bounce Pages */ 873 mtx_lock(&bounce_lock); 874 if (flags & BUS_DMA_NOWAIT) { 875 if (reserve_bounce_pages(dmat, map, 0) != 0) { 876 mtx_unlock(&bounce_lock); 877 return (ENOMEM); 878 } 879 } else { 880 if (reserve_bounce_pages(dmat, map, 1) != 0) { 881 /* Queue us for resources */ 882 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 883 mtx_unlock(&bounce_lock); 884 return (EINPROGRESS); 885 } 886 } 887 mtx_unlock(&bounce_lock); 888 889 return (0); 890} 891 892/* 893 * Add a single contiguous physical range to the segment list. 894 */ 895static int 896_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 897 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 898{ 899 bus_addr_t baddr, bmask; 900 int seg; 901 902 /* 903 * Make sure we don't cross any boundaries. 904 */ 905 bmask = ~(dmat->boundary - 1); 906 if (dmat->boundary > 0) { 907 baddr = (curaddr + dmat->boundary) & bmask; 908 if (sgsize > (baddr - curaddr)) 909 sgsize = (baddr - curaddr); 910 } 911 if (dmat->ranges) { 912 struct arm32_dma_range *dr; 913 914 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 915 curaddr); 916 if (dr == NULL) 917 return (0); 918 /* 919 * In a valid DMA range. Translate the physical 920 * memory address to an address in the DMA window. 921 */ 922 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 923 924 } 925 926 seg = *segp; 927 /* 928 * Insert chunk into a segment, coalescing with 929 * the previous segment if possible. 930 */ 931 if (seg >= 0 && 932 curaddr == segs[seg].ds_addr + segs[seg].ds_len && 933 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 934 (dmat->boundary == 0 || 935 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { 936 segs[seg].ds_len += sgsize; 937 } else { 938 if (++seg >= dmat->nsegments) 939 return (0); 940 segs[seg].ds_addr = curaddr; 941 segs[seg].ds_len = sgsize; 942 } 943 *segp = seg; 944 return (sgsize); 945} 946 947/* 948 * Utility function to load a physical buffer. segp contains 949 * the starting segment on entrace, and the ending segment on exit. 950 */ 951int 952_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 953 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 954{ 955 struct sync_list *sl; 956 bus_size_t sgsize; 957 bus_addr_t curaddr; 958 bus_addr_t sl_end = 0; 959 int error; 960 961 if (segs == NULL) 962 segs = dmat->segments; 963 964 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 965 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 966 if (map->pagesneeded != 0) { 967 error = _bus_dmamap_reserve_pages(dmat, map, flags); 968 if (error) 969 return (error); 970 } 971 } 972 973 sl = map->slist + map->sync_count - 1; 974 975 while (buflen > 0) { 976 curaddr = buf; 977 sgsize = MIN(buflen, dmat->maxsegsz); 978 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 979 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 980 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 981 curaddr = add_bounce_page(dmat, map, 0, curaddr, 982 sgsize); 983 } else { 984 if (map->sync_count > 0) 985 sl_end = VM_PAGE_TO_PHYS(sl->pages) + 986 sl->dataoffs + sl->datacount; 987 988 if (map->sync_count == 0 || curaddr != sl_end) { 989 if (++map->sync_count > dmat->nsegments) 990 break; 991 sl++; 992 sl->vaddr = 0; 993 sl->datacount = sgsize; 994 sl->pages = PHYS_TO_VM_PAGE(curaddr); 995 sl->dataoffs = curaddr & PAGE_MASK; 996 } else 997 sl->datacount += sgsize; 998 } 999 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1000 segp); 1001 if (sgsize == 0) 1002 break; 1003 buf += sgsize; 1004 buflen -= sgsize; 1005 } 1006 1007 /* 1008 * Did we fit? 1009 */ 1010 if (buflen != 0) { 1011 _bus_dmamap_unload(dmat, map); 1012 return (EFBIG); /* XXX better return value here? */ 1013 } 1014 return (0); 1015} 1016 1017int 1018_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, 1019 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 1020 bus_dma_segment_t *segs, int *segp) 1021{ 1022 1023 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, 1024 segs, segp)); 1025} 1026 1027/* 1028 * Utility function to load a linear buffer. segp contains 1029 * the starting segment on entrance, and the ending segment on exit. 1030 */ 1031int 1032_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 1033 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, 1034 int *segp) 1035{ 1036 bus_size_t sgsize; 1037 bus_addr_t curaddr; 1038 bus_addr_t sl_pend = 0; 1039 struct sync_list *sl; 1040 vm_offset_t kvaddr; 1041 vm_offset_t vaddr = (vm_offset_t)buf; 1042 vm_offset_t sl_vend = 0; 1043 int error = 0; 1044 1045 if (segs == NULL) 1046 segs = dmat->segments; 1047 if ((flags & BUS_DMA_LOAD_MBUF) != 0) 1048 map->flags |= DMAMAP_CACHE_ALIGNED; 1049 1050 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 1051 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 1052 if (map->pagesneeded != 0) { 1053 error = _bus_dmamap_reserve_pages(dmat, map, flags); 1054 if (error) 1055 return (error); 1056 } 1057 } 1058 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 1059 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 1060 1061 sl = map->slist + map->sync_count - 1; 1062 1063 while (buflen > 0) { 1064 /* 1065 * Get the physical address for this segment. 1066 */ 1067 if (__predict_true(pmap == kernel_pmap)) { 1068 curaddr = pmap_kextract(vaddr); 1069 kvaddr = vaddr; 1070 } else { 1071 curaddr = pmap_extract(pmap, vaddr); 1072 map->flags &= ~DMAMAP_COHERENT; 1073 kvaddr = 0; 1074 } 1075 1076 /* 1077 * Compute the segment size, and adjust counts. 1078 */ 1079 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 1080 if (sgsize > dmat->maxsegsz) 1081 sgsize = dmat->maxsegsz; 1082 if (buflen < sgsize) 1083 sgsize = buflen; 1084 1085 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 1086 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 1087 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 1088 sgsize); 1089 } else { 1090 if (map->sync_count > 0) { 1091 sl_pend = VM_PAGE_TO_PHYS(sl->pages) + 1092 sl->dataoffs + sl->datacount; 1093 sl_vend = sl->vaddr + sl->datacount; 1094 } 1095 1096 if (map->sync_count == 0 || 1097 (kvaddr != 0 && kvaddr != sl_vend) || 1098 (kvaddr == 0 && curaddr != sl_pend)) { 1099 1100 if (++map->sync_count > dmat->nsegments) 1101 goto cleanup; 1102 sl++; 1103 sl->vaddr = kvaddr; 1104 sl->datacount = sgsize; 1105 sl->pages = PHYS_TO_VM_PAGE(curaddr); 1106 sl->dataoffs = curaddr & PAGE_MASK; 1107 } else 1108 sl->datacount += sgsize; 1109 } 1110 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 1111 segp); 1112 if (sgsize == 0) 1113 break; 1114 vaddr += sgsize; 1115 buflen -= sgsize; 1116 } 1117 1118cleanup: 1119 /* 1120 * Did we fit? 1121 */ 1122 if (buflen != 0) { 1123 _bus_dmamap_unload(dmat, map); 1124 return (EFBIG); /* XXX better return value here? */ 1125 } 1126 return (0); 1127} 1128 1129void 1130__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, 1131 bus_dmamap_callback_t *callback, void *callback_arg) 1132{ 1133 1134 KASSERT(dmat != NULL, ("dmatag is NULL")); 1135 KASSERT(map != NULL, ("dmamap is NULL")); 1136 map->mem = *mem; 1137 map->callback = callback; 1138 map->callback_arg = callback_arg; 1139} 1140 1141bus_dma_segment_t * 1142_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 1143 bus_dma_segment_t *segs, int nsegs, int error) 1144{ 1145 1146 if (segs == NULL) 1147 segs = dmat->segments; 1148 return (segs); 1149} 1150 1151/* 1152 * Release the mapping held by map. 1153 */ 1154void 1155_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1156{ 1157 struct bounce_page *bpage; 1158 1159 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1160 STAILQ_REMOVE_HEAD(&map->bpages, links); 1161 free_bounce_page(dmat, bpage); 1162 } 1163 map->sync_count = 0; 1164} 1165 1166static void 1167bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, 1168 int bufaligned) 1169{ 1170 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1171 register_t s; 1172 int partial; 1173 1174 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1175 cpu_dcache_wb_range(buf, len); 1176 cpu_l2cache_wb_range(buf, len); 1177 } 1178 1179 /* 1180 * If the caller promises the buffer is properly aligned to a cache line 1181 * (even if the call parms make it look like it isn't) we can avoid 1182 * attempting to preserve the non-DMA part of the cache line in the 1183 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1184 * 1185 * This covers the case of mbufs, where we know how they're aligned and 1186 * know the CPU doesn't touch the header in front of the DMA data area 1187 * during the IO, but it may have touched it right before invoking the 1188 * sync, so a PREREAD writeback is required. 1189 * 1190 * It also handles buffers we created in bus_dmamem_alloc(), which are 1191 * always aligned and padded to cache line size even if the IO length 1192 * isn't a multiple of cache line size. In this case the PREREAD 1193 * writeback probably isn't required, but it's harmless. 1194 */ 1195 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1196 1197 if (op & BUS_DMASYNC_PREREAD) { 1198 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1199 cpu_dcache_inv_range(buf, len); 1200 cpu_l2cache_inv_range(buf, len); 1201 } else { 1202 cpu_dcache_wbinv_range(buf, len); 1203 cpu_l2cache_wbinv_range(buf, len); 1204 } 1205 } 1206 if (op & BUS_DMASYNC_POSTREAD) { 1207 if (partial && !bufaligned) { 1208 s = intr_disable(); 1209 if (buf & arm_dcache_align_mask) 1210 memcpy(_tmp_cl, (void *)(buf & 1211 ~arm_dcache_align_mask), 1212 buf & arm_dcache_align_mask); 1213 if ((buf + len) & arm_dcache_align_mask) 1214 memcpy(_tmp_clend, 1215 (void *)(buf + len), 1216 arm_dcache_align - 1217 ((buf + len) & arm_dcache_align_mask)); 1218 } 1219 cpu_dcache_inv_range(buf, len); 1220 cpu_l2cache_inv_range(buf, len); 1221 if (partial && !bufaligned) { 1222 if (buf & arm_dcache_align_mask) 1223 memcpy((void *)(buf & 1224 ~arm_dcache_align_mask), _tmp_cl, 1225 buf & arm_dcache_align_mask); 1226 if ((buf + len) & arm_dcache_align_mask) 1227 memcpy((void *)(buf + len), 1228 _tmp_clend, arm_dcache_align - 1229 ((buf + len) & arm_dcache_align_mask)); 1230 intr_restore(s); 1231 } 1232 } 1233} 1234 1235static void 1236bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, 1237 int bufaligned) 1238{ 1239 vm_offset_t tempvaddr; 1240 vm_page_t curpage; 1241 size_t npages; 1242 1243 if (sl->vaddr != 0) { 1244 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); 1245 return; 1246 } 1247 1248 tempvaddr = 0; 1249 npages = atop(round_page(sl->dataoffs + sl->datacount)); 1250 1251 for (curpage = sl->pages; curpage != sl->pages + npages; ++curpage) { 1252 /* 1253 * If the page is mapped to some other VA that hasn't 1254 * been supplied to busdma, then pmap_quick_enter_page() 1255 * will find all duplicate mappings and mark them 1256 * uncacheable. 1257 * That will also do any necessary wb/inv. Otherwise, 1258 * if the page is truly unmapped, then we don't actually 1259 * need to do cache maintenance. 1260 * XXX: May overwrite DMA'ed data in the POSTREAD 1261 * case where the CPU has written to a cacheline not 1262 * completely covered by the DMA region. 1263 */ 1264 KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) + 1265 ptoa(curpage - sl->pages), 1266 ("unexpected vm_page_t phys: 0x%08x != 0x%08x", 1267 VM_PAGE_TO_PHYS(curpage), VM_PAGE_TO_PHYS(sl->pages) + 1268 ptoa(curpage - sl->pages))); 1269 tempvaddr = pmap_quick_enter_page(curpage); 1270 pmap_quick_remove_page(tempvaddr); 1271 } 1272} 1273 1274static void 1275_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1276{ 1277 struct bounce_page *bpage; 1278 vm_offset_t datavaddr, tempvaddr; 1279 1280 if ((op & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) == 0) 1281 return; 1282 1283 STAILQ_FOREACH(bpage, &map->bpages, links) { 1284 tempvaddr = 0; 1285 datavaddr = bpage->datavaddr; 1286 if (op & BUS_DMASYNC_PREWRITE) { 1287 if (datavaddr == 0) { 1288 tempvaddr = 1289 pmap_quick_enter_page(bpage->datapage); 1290 datavaddr = tempvaddr | bpage->dataoffs; 1291 } 1292 bcopy((void *)datavaddr, 1293 (void *)bpage->vaddr, bpage->datacount); 1294 if (tempvaddr != 0) 1295 pmap_quick_remove_page(tempvaddr); 1296 cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); 1297 cpu_l2cache_wb_range(bpage->vaddr, bpage->datacount); 1298 dmat->bounce_zone->total_bounced++; 1299 } 1300 if (op & BUS_DMASYNC_POSTREAD) { 1301 cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); 1302 cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); 1303 if (datavaddr == 0) { 1304 tempvaddr = 1305 pmap_quick_enter_page(bpage->datapage); 1306 datavaddr = tempvaddr | bpage->dataoffs; 1307 } 1308 bcopy((void *)bpage->vaddr, 1309 (void *)datavaddr, bpage->datacount); 1310 if (tempvaddr != 0) 1311 pmap_quick_remove_page(tempvaddr); 1312 dmat->bounce_zone->total_bounced++; 1313 } 1314 } 1315} 1316 1317void 1318_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1319{ 1320 struct sync_list *sl, *end; 1321 int bufaligned; 1322 1323 if (op == BUS_DMASYNC_POSTWRITE) 1324 return; 1325 if (map->flags & DMAMAP_COHERENT) 1326 goto drain; 1327 if (STAILQ_FIRST(&map->bpages)) 1328 _bus_dmamap_sync_bp(dmat, map, op); 1329 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1330 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1331 if (map->sync_count) { 1332 end = &map->slist[map->sync_count]; 1333 for (sl = &map->slist[0]; sl != end; sl++) 1334 bus_dmamap_sync_sl(sl, op, bufaligned); 1335 } 1336 1337drain: 1338 1339 cpu_drain_writebuf(); 1340} 1341 1342static void 1343init_bounce_pages(void *dummy __unused) 1344{ 1345 1346 total_bpages = 0; 1347 STAILQ_INIT(&bounce_zone_list); 1348 STAILQ_INIT(&bounce_map_waitinglist); 1349 STAILQ_INIT(&bounce_map_callbacklist); 1350 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1351} 1352SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1353 1354static struct sysctl_ctx_list * 1355busdma_sysctl_tree(struct bounce_zone *bz) 1356{ 1357 1358 return (&bz->sysctl_tree); 1359} 1360 1361static struct sysctl_oid * 1362busdma_sysctl_tree_top(struct bounce_zone *bz) 1363{ 1364 1365 return (bz->sysctl_tree_top); 1366} 1367 1368static int 1369alloc_bounce_zone(bus_dma_tag_t dmat) 1370{ 1371 struct bounce_zone *bz; 1372 1373 /* Check to see if we already have a suitable zone */ 1374 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1375 if ((dmat->alignment <= bz->alignment) && 1376 (dmat->lowaddr >= bz->lowaddr)) { 1377 dmat->bounce_zone = bz; 1378 return (0); 1379 } 1380 } 1381 1382 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1383 M_NOWAIT | M_ZERO)) == NULL) 1384 return (ENOMEM); 1385 1386 STAILQ_INIT(&bz->bounce_page_list); 1387 bz->free_bpages = 0; 1388 bz->reserved_bpages = 0; 1389 bz->active_bpages = 0; 1390 bz->lowaddr = dmat->lowaddr; 1391 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1392 bz->map_count = 0; 1393 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1394 busdma_zonecount++; 1395 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1396 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1397 dmat->bounce_zone = bz; 1398 1399 sysctl_ctx_init(&bz->sysctl_tree); 1400 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1401 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1402 CTLFLAG_RD, 0, ""); 1403 if (bz->sysctl_tree_top == NULL) { 1404 sysctl_ctx_free(&bz->sysctl_tree); 1405 return (0); /* XXX error code? */ 1406 } 1407 1408 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1409 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1410 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1411 "Total bounce pages"); 1412 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1413 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1414 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1415 "Free bounce pages"); 1416 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1417 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1418 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1419 "Reserved bounce pages"); 1420 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1421 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1422 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1423 "Active bounce pages"); 1424 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1425 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1426 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1427 "Total bounce requests (pages bounced)"); 1428 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1429 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1430 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1431 "Total bounce requests that were deferred"); 1432 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1433 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1434 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1435 SYSCTL_ADD_ULONG(busdma_sysctl_tree(bz), 1436 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1437 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1438 1439 return (0); 1440} 1441 1442static int 1443alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1444{ 1445 struct bounce_zone *bz; 1446 int count; 1447 1448 bz = dmat->bounce_zone; 1449 count = 0; 1450 while (numpages > 0) { 1451 struct bounce_page *bpage; 1452 1453 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1454 M_NOWAIT | M_ZERO); 1455 1456 if (bpage == NULL) 1457 break; 1458 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1459 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1460 if (bpage->vaddr == 0) { 1461 free(bpage, M_DEVBUF); 1462 break; 1463 } 1464 bpage->busaddr = pmap_kextract(bpage->vaddr); 1465 mtx_lock(&bounce_lock); 1466 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1467 total_bpages++; 1468 bz->total_bpages++; 1469 bz->free_bpages++; 1470 mtx_unlock(&bounce_lock); 1471 count++; 1472 numpages--; 1473 } 1474 return (count); 1475} 1476 1477static int 1478reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1479{ 1480 struct bounce_zone *bz; 1481 int pages; 1482 1483 mtx_assert(&bounce_lock, MA_OWNED); 1484 bz = dmat->bounce_zone; 1485 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1486 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1487 return (map->pagesneeded - (map->pagesreserved + pages)); 1488 bz->free_bpages -= pages; 1489 bz->reserved_bpages += pages; 1490 map->pagesreserved += pages; 1491 pages = map->pagesneeded - map->pagesreserved; 1492 1493 return (pages); 1494} 1495 1496static bus_addr_t 1497add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1498 bus_addr_t addr, bus_size_t size) 1499{ 1500 struct bounce_zone *bz; 1501 struct bounce_page *bpage; 1502 1503 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1504 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1505 1506 bz = dmat->bounce_zone; 1507 if (map->pagesneeded == 0) 1508 panic("add_bounce_page: map doesn't need any pages"); 1509 map->pagesneeded--; 1510 1511 if (map->pagesreserved == 0) 1512 panic("add_bounce_page: map doesn't need any pages"); 1513 map->pagesreserved--; 1514 1515 mtx_lock(&bounce_lock); 1516 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1517 if (bpage == NULL) 1518 panic("add_bounce_page: free page list is empty"); 1519 1520 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1521 bz->reserved_bpages--; 1522 bz->active_bpages++; 1523 mtx_unlock(&bounce_lock); 1524 1525 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1526 /* Page offset needs to be preserved. */ 1527 bpage->vaddr |= addr & PAGE_MASK; 1528 bpage->busaddr |= addr & PAGE_MASK; 1529 } 1530 bpage->datavaddr = vaddr; 1531 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1532 bpage->dataoffs = addr & PAGE_MASK; 1533 bpage->datacount = size; 1534 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1535 return (bpage->busaddr); 1536} 1537 1538static void 1539free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1540{ 1541 struct bus_dmamap *map; 1542 struct bounce_zone *bz; 1543 1544 bz = dmat->bounce_zone; 1545 bpage->datavaddr = 0; 1546 bpage->datacount = 0; 1547 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1548 /* 1549 * Reset the bounce page to start at offset 0. Other uses 1550 * of this bounce page may need to store a full page of 1551 * data and/or assume it starts on a page boundary. 1552 */ 1553 bpage->vaddr &= ~PAGE_MASK; 1554 bpage->busaddr &= ~PAGE_MASK; 1555 } 1556 1557 mtx_lock(&bounce_lock); 1558 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1559 bz->free_bpages++; 1560 bz->active_bpages--; 1561 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1562 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1563 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1564 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1565 map, links); 1566 busdma_swi_pending = 1; 1567 bz->total_deferred++; 1568 swi_sched(vm_ih, 0); 1569 } 1570 } 1571 mtx_unlock(&bounce_lock); 1572} 1573 1574void 1575busdma_swi(void) 1576{ 1577 bus_dma_tag_t dmat; 1578 struct bus_dmamap *map; 1579 1580 mtx_lock(&bounce_lock); 1581 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1582 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1583 mtx_unlock(&bounce_lock); 1584 dmat = map->dmat; 1585 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_LOCK); 1586 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, 1587 map->callback_arg, BUS_DMA_WAITOK); 1588 dmat->lockfunc(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1589 mtx_lock(&bounce_lock); 1590 } 1591 mtx_unlock(&bounce_lock); 1592} 1593