busdma_machdep-v4.c revision 132514
1/* 2 * Copyright (c) 2004 Olivier Houchard 3 * Copyright (c) 2002 Peter Grehan 4 * Copyright (c) 1997, 1998 Justin T. Gibbs. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions, and the following disclaimer, 12 * without modification, immediately at the beginning of the file. 13 * 2. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 132514 2004-07-21 22:04:05Z cognet $"); 33 34/* 35 * MacPPC bus dma support routines 36 */ 37 38#define _ARM32_BUS_DMA_PRIVATE 39#include <sys/param.h> 40#include <sys/systm.h> 41#include <sys/malloc.h> 42#include <sys/bus.h> 43#include <sys/interrupt.h> 44#include <sys/lock.h> 45#include <sys/proc.h> 46#include <sys/mutex.h> 47#include <sys/mbuf.h> 48#include <sys/uio.h> 49 50#include <vm/vm.h> 51#include <vm/vm_page.h> 52#include <vm/vm_map.h> 53 54#include <machine/atomic.h> 55#include <machine/bus.h> 56#include <machine/cpufunc.h> 57 58struct bus_dma_tag { 59 bus_dma_tag_t parent; 60 bus_size_t alignment; 61 bus_size_t boundary; 62 bus_addr_t lowaddr; 63 bus_addr_t highaddr; 64 bus_dma_filter_t *filter; 65 void *filterarg; 66 bus_size_t maxsize; 67 u_int nsegments; 68 bus_size_t maxsegsz; 69 int flags; 70 int ref_count; 71 int map_count; 72 bus_dma_lock_t *lockfunc; 73 void *lockfuncarg; 74 /* 75 * DMA range for this tag. If the page doesn't fall within 76 * one of these ranges, an error is returned. The caller 77 * may then decide what to do with the transfer. If the 78 * range pointer is NULL, it is ignored. 79 */ 80 struct arm32_dma_range *ranges; 81 int _nranges; 82 83}; 84 85struct arm_seglist { 86 bus_dma_segment_t seg; 87 SLIST_ENTRY(arm_seglist) next; 88}; 89 90#define MAX_SEGS 512 91struct bus_dmamap { 92 bus_dma_tag_t dmat; 93 int flags; 94 SLIST_HEAD(, arm_seglist) seglist; 95}; 96 97/* 98 * Check to see if the specified page is in an allowed DMA range. 99 */ 100 101static int 102bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t segs[], 103 bus_dmamap_t map, void *buf, bus_size_t buflen, struct thread *td, 104 int flags, vm_offset_t *lastaddrp, int *segp, 105 int first); 106static __inline struct arm32_dma_range * 107_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 108 bus_addr_t curaddr) 109{ 110 struct arm32_dma_range *dr; 111 int i; 112 113 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 114 if (curaddr >= dr->dr_sysbase && 115 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 116 return (dr); 117 } 118 119 return (NULL); 120} 121/* 122 * Convenience function for manipulating driver locks from busdma (during 123 * busdma_swi, for example). Drivers that don't provide their own locks 124 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 125 * non-mutex locking scheme don't have to use this at all. 126 */ 127void 128busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 129{ 130 struct mtx *dmtx; 131 132 dmtx = (struct mtx *)arg; 133 switch (op) { 134 case BUS_DMA_LOCK: 135 mtx_lock(dmtx); 136 break; 137 case BUS_DMA_UNLOCK: 138 mtx_unlock(dmtx); 139 break; 140 default: 141 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 142 } 143} 144 145/* 146 * dflt_lock should never get called. It gets put into the dma tag when 147 * lockfunc == NULL, which is only valid if the maps that are associated 148 * with the tag are meant to never be defered. 149 * XXX Should have a way to identify which driver is responsible here. 150 */ 151static void 152dflt_lock(void *arg, bus_dma_lock_op_t op) 153{ 154#ifdef INVARIANTS 155 panic("driver error: busdma dflt_lock called"); 156#else 157 printf("DRIVER_ERROR: busdma dflt_lock called\n"); 158#endif 159} 160 161/* 162 * Allocate a device specific dma_tag. 163 */ 164int 165bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 166 bus_size_t boundary, bus_addr_t lowaddr, 167 bus_addr_t highaddr, bus_dma_filter_t *filter, 168 void *filterarg, bus_size_t maxsize, int nsegments, 169 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 170 void *lockfuncarg, bus_dma_tag_t *dmat) 171{ 172 bus_dma_tag_t newtag; 173 int error = 0; 174 175 /* Return a NULL tag on failure */ 176 *dmat = NULL; 177 178 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 179 if (newtag == NULL) 180 return (ENOMEM); 181 182 newtag->parent = parent; 183 newtag->alignment = alignment; 184 newtag->boundary = boundary; 185 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 186 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 187 newtag->filter = filter; 188 newtag->filterarg = filterarg; 189 newtag->maxsize = maxsize; 190 newtag->nsegments = nsegments; 191 newtag->maxsegsz = maxsegsz; 192 newtag->flags = flags; 193 newtag->ref_count = 1; /* Count ourself */ 194 newtag->map_count = 0; 195 newtag->ranges = bus_dma_get_range(); 196 if (lockfunc != NULL) { 197 newtag->lockfunc = lockfunc; 198 newtag->lockfuncarg = lockfuncarg; 199 } else { 200 newtag->lockfunc = dflt_lock; 201 newtag->lockfuncarg = NULL; 202 } 203 204 /* 205 * Take into account any restrictions imposed by our parent tag 206 */ 207 if (parent != NULL) { 208 newtag->lowaddr = min(parent->lowaddr, newtag->lowaddr); 209 newtag->highaddr = max(parent->highaddr, newtag->highaddr); 210 211 /* 212 * XXX Not really correct??? Probably need to honor boundary 213 * all the way up the inheritence chain. 214 */ 215 newtag->boundary = max(parent->boundary, newtag->boundary); 216 if (newtag->filter == NULL) { 217 /* 218 * Short circuit looking at our parent directly 219 * since we have encapsulated all of its information 220 */ 221 newtag->filter = parent->filter; 222 newtag->filterarg = parent->filterarg; 223 newtag->parent = parent->parent; 224 } 225 if (newtag->parent != NULL) 226 atomic_add_int(&parent->ref_count, 1); 227 } 228 229 *dmat = newtag; 230 return (error); 231} 232 233int 234bus_dma_tag_destroy(bus_dma_tag_t dmat) 235{ 236 if (dmat != NULL) { 237 238 if (dmat->map_count != 0) 239 return (EBUSY); 240 241 while (dmat != NULL) { 242 bus_dma_tag_t parent; 243 244 parent = dmat->parent; 245 atomic_subtract_int(&dmat->ref_count, 1); 246 if (dmat->ref_count == 0) { 247 free(dmat, M_DEVBUF); 248 /* 249 * Last reference count, so 250 * release our reference 251 * count on our parent. 252 */ 253 dmat = parent; 254 } else 255 dmat = NULL; 256 } 257 } 258 return (0); 259} 260 261static void 262arm_dmamap_freesegs(bus_dmamap_t map) 263{ 264 struct arm_seglist *seg = SLIST_FIRST(&map->seglist); 265 266 while (seg) { 267 struct arm_seglist *next; 268 269 next = SLIST_NEXT(seg, next); 270 SLIST_REMOVE_HEAD(&map->seglist, next); 271 free(seg, M_DEVBUF); 272 seg = next; 273 } 274} 275 276static int 277arm_dmamap_addseg(bus_dmamap_t map, vm_offset_t addr, vm_size_t size) 278{ 279 struct arm_seglist *seg = malloc(sizeof(*seg), M_DEVBUF, M_NOWAIT); 280 281 if (!seg) 282 return (ENOMEM); 283 seg->seg.ds_addr = addr; 284 seg->seg.ds_len = size; 285 SLIST_INSERT_HEAD(&map->seglist, seg, next); 286 return (0); 287} 288 289/* 290 * Allocate a handle for mapping from kva/uva/physical 291 * address space into bus device space. 292 */ 293int 294bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 295{ 296 bus_dmamap_t newmap; 297 298 newmap = malloc(sizeof(*newmap), M_DEVBUF, M_NOWAIT | M_ZERO); 299 if (newmap == NULL) 300 return (ENOMEM); 301 SLIST_INIT(&newmap->seglist); 302 *mapp = newmap; 303 dmat->map_count++; 304 305 return (0); 306} 307 308/* 309 * Destroy a handle for mapping from kva/uva/physical 310 * address space into bus device space. 311 */ 312int 313bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 314{ 315 arm_dmamap_freesegs(map); 316 free(map, M_DEVBUF); 317 dmat->map_count--; 318 return (0); 319} 320 321/* 322 * Allocate a piece of memory that can be efficiently mapped into 323 * bus device space based on the constraints lited in the dma tag. 324 * A dmamap to for use with dmamap_load is also allocated. 325 */ 326int 327bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 328 bus_dmamap_t *mapp) 329{ 330 bus_dmamap_t newmap; 331 332 int mflags; 333 334 if (flags & BUS_DMA_NOWAIT) 335 mflags = M_NOWAIT; 336 else 337 mflags = M_WAITOK; 338 if (flags & BUS_DMA_ZERO) 339 mflags |= M_ZERO; 340 341 newmap = malloc(sizeof(*newmap), M_DEVBUF, M_NOWAIT | M_ZERO); 342 if (newmap == NULL) 343 return (ENOMEM); 344 SLIST_INIT(&newmap->seglist); 345 *mapp = newmap; 346 if (dmat->maxsize <= PAGE_SIZE) { 347 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 348 } else { 349 /* 350 * XXX Use Contigmalloc until it is merged into this facility 351 * and handles multi-seg allocations. Nobody is doing 352 * multi-seg allocations yet though. 353 */ 354 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 355 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 356 dmat->boundary); 357 } 358 359 if (*vaddr == NULL) { 360 free(newmap, M_DEVBUF); 361 *mapp = NULL; 362 return (ENOMEM); 363 } 364 365 return (0); 366} 367 368/* 369 * Free a piece of memory and it's allocated dmamap, that was allocated 370 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 371 */ 372void 373bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 374{ 375 if (map != NULL) 376 panic("bus_dmamem_free: Invalid map freed\n"); 377 if (dmat->maxsize <= PAGE_SIZE) 378 free(vaddr, M_DEVBUF); 379 else { 380 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 381 } 382 arm_dmamap_freesegs(map); 383 free(map, M_DEVBUF); 384} 385 386/* 387 * Map the buffer buf into bus space using the dmamap map. 388 */ 389int 390bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 391 bus_size_t buflen, bus_dmamap_callback_t *callback, 392 void *callback_arg, int flags) 393{ 394 vm_offset_t lastaddr = 0; 395 int error, nsegs = 0; 396#ifdef __GNUC__ 397 bus_dma_segment_t dm_segments[dmat->nsegments]; 398#else 399 bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 400#endif 401 402 error = bus_dmamap_load_buffer(dmat, 403 dm_segments, map, buf, buflen, NULL, 404 flags, &lastaddr, &nsegs, 1); 405 (*callback)(callback_arg, dm_segments, nsegs, error); 406 407 return (0); 408} 409 410/* 411 * Utility function to load a linear buffer. lastaddrp holds state 412 * between invocations (for multiple-buffer loads). segp contains 413 * the starting segment on entrance, and the ending segment on exit. 414 * first indicates if this is the first invocation of this function. 415 */ 416static int 417bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t segs[], 418 bus_dmamap_t map, void *buf, bus_size_t buflen, struct thread *td, 419 int flags, vm_offset_t *lastaddrp, int *segp, 420 int first) 421{ 422 bus_size_t sgsize; 423 bus_addr_t curaddr, lastaddr, baddr, bmask; 424 vm_offset_t vaddr = (vm_offset_t)buf; 425 int seg; 426 int error = 0; 427 pmap_t pmap; 428 pd_entry_t *pde; 429 pt_entry_t pte; 430 pt_entry_t *ptep; 431 432 433 if (td != NULL) 434 pmap = vmspace_pmap(td->td_proc->p_vmspace); 435 else 436 pmap = pmap_kernel(); 437 438 lastaddr = *lastaddrp; 439 bmask = ~(dmat->boundary - 1); 440 441 for (seg = *segp; buflen > 0 ; ) { 442 /* 443 * Get the physical address for this segment. 444 * 445 * XXX Don't support checking for coherent mappings 446 * XXX in user address space. 447 */ 448 if (__predict_true(pmap == pmap_kernel())) { 449 (void) pmap_get_pde_pte(pmap, vaddr, &pde, &ptep); 450 if (__predict_false(pmap_pde_section(pde))) { 451 curaddr = (*pde & L1_S_FRAME) | 452 (vaddr & L1_S_OFFSET); 453 if (*pde & L1_S_CACHE_MASK) { 454 map->flags &= 455 ~ARM32_DMAMAP_COHERENT; 456 } 457 } else { 458 pte = *ptep; 459 KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV, 460 ("INV type")); 461 if (__predict_false((pte & L2_TYPE_MASK) 462 == L2_TYPE_L)) { 463 curaddr = (pte & L2_L_FRAME) | 464 (vaddr & L2_L_OFFSET); 465 if (pte & L2_L_CACHE_MASK) { 466 map->flags &= 467 ~ARM32_DMAMAP_COHERENT; 468 } 469 } else { 470 curaddr = (pte & L2_S_FRAME) | 471 (vaddr & L2_S_OFFSET); 472 if (pte & L2_S_CACHE_MASK) { 473 map->flags &= 474 ~ARM32_DMAMAP_COHERENT; 475 } 476 } 477 } 478 } else { 479 curaddr = pmap_extract(pmap, vaddr); 480 map->flags &= ~ARM32_DMAMAP_COHERENT; 481 } 482 483 /* 484 * Compute the segment size, and adjust counts. 485 */ 486 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 487 if (buflen < sgsize) 488 sgsize = buflen; 489 490 /* 491 * Make sure we don't cross any boundaries. 492 */ 493 if (dmat->boundary > 0) { 494 baddr = (curaddr + dmat->boundary) & bmask; 495 if (sgsize > (baddr - curaddr)) 496 sgsize = (baddr - curaddr); 497 } 498 499 /* 500 * Insert chunk into a segment, coalescing with 501 * the previous segment if possible. 502 */ 503 error = arm_dmamap_addseg(map, 504 (vm_offset_t)curaddr, sgsize); 505 if (error) 506 break; 507 508 if (first) { 509 segs[seg].ds_addr = curaddr; 510 segs[seg].ds_len = sgsize; 511 first = 0; 512 } else { 513 if (curaddr == lastaddr && 514 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 515 (dmat->boundary == 0 || 516 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 517 segs[seg].ds_len += sgsize; 518 else { 519 if (++seg >= dmat->nsegments) 520 break; 521 segs[seg].ds_addr = curaddr; 522 segs[seg].ds_len = sgsize; 523 } 524 } 525 526 lastaddr = curaddr + sgsize; 527 vaddr += sgsize; 528 buflen -= sgsize; 529 } 530 531 *segp = seg; 532 *lastaddrp = lastaddr; 533 534 /* 535 * Did we fit? 536 */ 537 if (buflen != 0) 538 error = EFBIG; /* XXX better return value here? */ 539 return (error); 540} 541 542/* 543 * Like bus_dmamap_load(), but for mbufs. 544 */ 545int 546bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 547 bus_dmamap_callback2_t *callback, void *callback_arg, 548 int flags) 549{ 550#ifdef __GNUC__ 551 bus_dma_segment_t dm_segments[dmat->nsegments]; 552#else 553 bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 554#endif 555 int nsegs = 0, error = 0; 556 557 M_ASSERTPKTHDR(m0); 558 559 if (m0->m_pkthdr.len <= dmat->maxsize) { 560 int first = 1; 561 vm_offset_t lastaddr = 0; 562 struct mbuf *m; 563 564 for (m = m0; m != NULL && error == 0; m = m->m_next) { 565 if (m->m_len > 0) { 566 error = bus_dmamap_load_buffer(dmat, 567 dm_segments, map, m->m_data, m->m_len, NULL, 568 flags, &lastaddr, &nsegs, first); 569 first = 0; 570 } 571 } 572 } else { 573 error = EINVAL; 574 } 575 576 if (error) { 577 /* 578 * force "no valid mappings" on error in callback. 579 */ 580 (*callback)(callback_arg, dm_segments, 0, 0, error); 581 } else { 582 (*callback)(callback_arg, dm_segments, nsegs+1, 583 m0->m_pkthdr.len, error); 584 } 585 return (error); 586} 587 588/* 589 * Like bus_dmamap_load(), but for uios. 590 */ 591int 592bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 593 bus_dmamap_callback2_t *callback, void *callback_arg, 594 int flags) 595{ 596 vm_offset_t lastaddr; 597#ifdef __GNUC__ 598 bus_dma_segment_t dm_segments[dmat->nsegments]; 599#else 600 bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 601#endif 602 int nsegs, i, error, first; 603 bus_size_t resid; 604 struct iovec *iov; 605 struct thread *td = NULL; 606 607 resid = uio->uio_resid; 608 iov = uio->uio_iov; 609 610 if (uio->uio_segflg == UIO_USERSPACE) { 611 td = uio->uio_td; 612 KASSERT(td != NULL, 613 ("bus_dmamap_load_uio: USERSPACE but no proc")); 614 } 615 616 first = 1; 617 nsegs = error = 0; 618 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 619 /* 620 * Now at the first iovec to load. Load each iovec 621 * until we have exhausted the residual count. 622 */ 623 bus_size_t minlen = 624 resid < iov[i].iov_len ? resid : iov[i].iov_len; 625 caddr_t addr = (caddr_t) iov[i].iov_base; 626 627 if (minlen > 0) { 628 error = bus_dmamap_load_buffer(dmat, dm_segments, map, 629 addr, minlen, td, flags, &lastaddr, &nsegs, first); 630 631 first = 0; 632 633 resid -= minlen; 634 } 635 } 636 637 if (error) { 638 /* 639 * force "no valid mappings" on error in callback. 640 */ 641 (*callback)(callback_arg, dm_segments, 0, 0, error); 642 } else { 643 (*callback)(callback_arg, dm_segments, nsegs+1, 644 uio->uio_resid, error); 645 } 646 647 return (error); 648} 649 650/* 651 * Release the mapping held by map. A no-op on PowerPC. 652 */ 653void 654bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 655{ 656 arm_dmamap_freesegs(map); 657 return; 658} 659 660void 661bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 662{ 663 struct arm_seglist *seg = SLIST_FIRST(&map->seglist); 664 665 if (op != BUS_DMASYNC_PREREAD && op != BUS_DMASYNC_PREWRITE) 666 return; 667 /* Skip cache frobbing if mapping was COHERENT. */ 668 if (map->flags & ARM32_DMAMAP_COHERENT) { 669 /* Drain the write buffer. */ 670 cpu_drain_writebuf(); 671 return; 672 } 673 while (seg) { 674 cpu_dcache_wbinv_range(seg->seg.ds_addr, seg->seg.ds_len); 675 seg = SLIST_NEXT(seg, next); 676 } 677} 678