busdma_machdep.c revision 113347
1104477Ssam/* 2104477Ssam * Copyright (c) 2002 Peter Grehan 3104477Ssam * Copyright (c) 1997, 1998 Justin T. Gibbs. 4104477Ssam * All rights reserved. 5104477Ssam * 6104477Ssam * Redistribution and use in source and binary forms, with or without 7104477Ssam * modification, are permitted provided that the following conditions 8104477Ssam * are met: 9104477Ssam * 1. Redistributions of source code must retain the above copyright 10104477Ssam * notice, this list of conditions, and the following disclaimer, 11104477Ssam * without modification, immediately at the beginning of the file. 12104477Ssam * 2. The name of the author may not be used to endorse or promote products 13104477Ssam * derived from this software without specific prior written permission. 14104477Ssam * 15104477Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16104477Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17104477Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18104477Ssam * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 19104477Ssam * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20104477Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21104477Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22104477Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23104477Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24104477Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25104477Ssam * SUCH DAMAGE. 26104477Ssam * 27104477Ssam * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 28104477Ssam */ 29104477Ssam 30104477Ssam#include <sys/cdefs.h> 31104477Ssam__FBSDID("$FreeBSD: head/sys/powerpc/powerpc/busdma_machdep.c 113347 2003-04-10 23:03:33Z mux $"); 32104477Ssam 33104477Ssam/* 34104477Ssam * MacPPC bus dma support routines 35104477Ssam */ 36104477Ssam 37104477Ssam#include <sys/param.h> 38104477Ssam#include <sys/systm.h> 39104477Ssam#include <sys/malloc.h> 40104477Ssam#include <sys/bus.h> 41104477Ssam#include <sys/interrupt.h> 42104477Ssam#include <sys/lock.h> 43104477Ssam#include <sys/proc.h> 44104477Ssam#include <sys/mutex.h> 45104477Ssam#include <sys/mbuf.h> 46104477Ssam#include <sys/uio.h> 47104477Ssam 48104477Ssam#include <vm/vm.h> 49104477Ssam#include <vm/vm_page.h> 50104477Ssam#include <vm/vm_map.h> 51104477Ssam 52104477Ssam#include <machine/atomic.h> 53104477Ssam#include <machine/bus.h> 54104477Ssam#include <machine/cpufunc.h> 55104477Ssam 56104477Ssamstruct bus_dma_tag { 57104477Ssam bus_dma_tag_t parent; 58104477Ssam bus_size_t alignment; 59104477Ssam bus_size_t boundary; 60104477Ssam bus_addr_t lowaddr; 61104477Ssam bus_addr_t highaddr; 62104477Ssam bus_dma_filter_t *filter; 63104477Ssam void *filterarg; 64104477Ssam bus_size_t maxsize; 65104477Ssam u_int nsegments; 66104477Ssam bus_size_t maxsegsz; 67104477Ssam int flags; 68104477Ssam int ref_count; 69104477Ssam int map_count; 70104477Ssam}; 71104477Ssam 72104477Ssamstruct bus_dmamap { 73104477Ssam bus_dma_tag_t dmat; 74104477Ssam void *buf; /* unmapped buffer pointer */ 75104477Ssam bus_size_t buflen; /* unmapped buffer length */ 76104477Ssam bus_dmamap_callback_t *callback; 77104477Ssam void *callback_arg; 78104477Ssam}; 79104477Ssam 80104477Ssam/* 81104477Ssam * Allocate a device specific dma_tag. 82104477Ssam */ 83104477Ssamint 84104477Ssambus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 85104477Ssam bus_size_t boundary, bus_addr_t lowaddr, 86104477Ssam bus_addr_t highaddr, bus_dma_filter_t *filter, 87104477Ssam void *filterarg, bus_size_t maxsize, int nsegments, 88104477Ssam bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat) 89104477Ssam{ 90104477Ssam bus_dma_tag_t newtag; 91104477Ssam int error = 0; 92104477Ssam 93104477Ssam /* Return a NULL tag on failure */ 94104477Ssam *dmat = NULL; 95104477Ssam 96104477Ssam newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 97104477Ssam if (newtag == NULL) 98104477Ssam return (ENOMEM); 99104477Ssam 100104477Ssam newtag->parent = parent; 101104477Ssam newtag->alignment = alignment; 102104477Ssam newtag->boundary = boundary; 103104477Ssam newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 104104477Ssam newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 105104477Ssam newtag->filter = filter; 106104477Ssam newtag->filterarg = filterarg; 107104477Ssam newtag->maxsize = maxsize; 108104477Ssam newtag->nsegments = nsegments; 109104477Ssam newtag->maxsegsz = maxsegsz; 110104477Ssam newtag->flags = flags; 111105251Smarkm newtag->ref_count = 1; /* Count ourself */ 112104477Ssam newtag->map_count = 0; 113104477Ssam 114104477Ssam /* 115104477Ssam * Take into account any restrictions imposed by our parent tag 116104477Ssam */ 117104477Ssam if (parent != NULL) { 118104477Ssam newtag->lowaddr = min(parent->lowaddr, newtag->lowaddr); 119104477Ssam newtag->highaddr = max(parent->highaddr, newtag->highaddr); 120104477Ssam 121104477Ssam /* 122104477Ssam * XXX Not really correct??? Probably need to honor boundary 123104477Ssam * all the way up the inheritence chain. 124104477Ssam */ 125104477Ssam newtag->boundary = max(parent->boundary, newtag->boundary); 126104477Ssam if (newtag->filter == NULL) { 127104477Ssam /* 128104477Ssam * Short circuit looking at our parent directly 129104477Ssam * since we have encapsulated all of its information 130104477Ssam */ 131104477Ssam newtag->filter = parent->filter; 132104477Ssam newtag->filterarg = parent->filterarg; 133104477Ssam newtag->parent = parent->parent; 134104477Ssam } 135104477Ssam if (newtag->parent != NULL) 136104477Ssam atomic_add_int(&parent->ref_count, 1); 137104477Ssam } 138104477Ssam 139104477Ssam *dmat = newtag; 140104477Ssam return (error); 141104477Ssam} 142104477Ssam 143104477Ssamint 144104477Ssambus_dma_tag_destroy(bus_dma_tag_t dmat) 145104477Ssam{ 146104477Ssam if (dmat != NULL) { 147104477Ssam 148104477Ssam if (dmat->map_count != 0) 149104477Ssam return (EBUSY); 150104477Ssam 151104477Ssam while (dmat != NULL) { 152104477Ssam bus_dma_tag_t parent; 153104477Ssam 154104477Ssam parent = dmat->parent; 155104477Ssam atomic_subtract_int(&dmat->ref_count, 1); 156104477Ssam if (dmat->ref_count == 0) { 157104477Ssam free(dmat, M_DEVBUF); 158104477Ssam /* 159104477Ssam * Last reference count, so 160104477Ssam * release our reference 161104477Ssam * count on our parent. 162104477Ssam */ 163104477Ssam dmat = parent; 164104477Ssam } else 165104477Ssam dmat = NULL; 166104477Ssam } 167104477Ssam } 168104477Ssam return (0); 169104477Ssam} 170104477Ssam 171104477Ssam/* 172104477Ssam * Allocate a handle for mapping from kva/uva/physical 173104477Ssam * address space into bus device space. 174104477Ssam */ 175104477Ssamint 176104477Ssambus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 177104477Ssam{ 178104477Ssam *mapp = NULL; 179104477Ssam dmat->map_count++; 180104477Ssam 181104477Ssam return (0); 182104477Ssam} 183104477Ssam 184104477Ssam/* 185104477Ssam * Destroy a handle for mapping from kva/uva/physical 186104477Ssam * address space into bus device space. 187104477Ssam */ 188104477Ssamint 189104477Ssambus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 190104477Ssam{ 191104477Ssam if (map != NULL) { 192104477Ssam panic("dmamap_destroy: NULL?\n"); 193104477Ssam } 194104477Ssam dmat->map_count--; 195104477Ssam return (0); 196104477Ssam} 197104477Ssam 198104477Ssam/* 199104477Ssam * Allocate a piece of memory that can be efficiently mapped into 200104477Ssam * bus device space based on the constraints lited in the dma tag. 201104477Ssam * A dmamap to for use with dmamap_load is also allocated. 202104477Ssam */ 203104477Ssamint 204104477Ssambus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 205104477Ssam bus_dmamap_t *mapp) 206104477Ssam{ 207104477Ssam *mapp = NULL; 208104477Ssam 209104477Ssam if (dmat->maxsize <= PAGE_SIZE) { 210104477Ssam *vaddr = malloc(dmat->maxsize, M_DEVBUF, 211104477Ssam (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK); 212104477Ssam } else { 213104477Ssam /* 214104477Ssam * XXX Use Contigmalloc until it is merged into this facility 215104477Ssam * and handles multi-seg allocations. Nobody is doing 216104477Ssam * multi-seg allocations yet though. 217104477Ssam */ 218104477Ssam mtx_lock(&Giant); 219104477Ssam *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, 220104477Ssam (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK, 221104477Ssam 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 222104477Ssam dmat->boundary); 223104477Ssam mtx_unlock(&Giant); 224104477Ssam } 225104477Ssam 226104477Ssam if (*vaddr == NULL) 227104477Ssam return (ENOMEM); 228104477Ssam 229104477Ssam return (0); 230104477Ssam} 231104477Ssam 232104477Ssam/* 233104477Ssam * Free a piece of memory and it's allocated dmamap, that was allocated 234104477Ssam * via bus_dmamem_alloc. Make the same choice for free/contigfree. 235104477Ssam */ 236104477Ssamvoid 237104477Ssambus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 238104477Ssam{ 239104477Ssam if (map != NULL) 240104477Ssam panic("bus_dmamem_free: Invalid map freed\n"); 241104477Ssam if (dmat->maxsize <= PAGE_SIZE) 242104477Ssam free(vaddr, M_DEVBUF); 243104477Ssam else { 244104477Ssam mtx_lock(&Giant); 245104477Ssam contigfree(vaddr, dmat->maxsize, M_DEVBUF); 246104477Ssam mtx_unlock(&Giant); 247104477Ssam } 248104477Ssam} 249104477Ssam 250104477Ssam/* 251104477Ssam * Map the buffer buf into bus space using the dmamap map. 252104477Ssam */ 253104477Ssamint 254104477Ssambus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 255104477Ssam bus_size_t buflen, bus_dmamap_callback_t *callback, 256104477Ssam void *callback_arg, int flags) 257104477Ssam{ 258104477Ssam vm_offset_t vaddr; 259104477Ssam vm_offset_t paddr; 260104477Ssam#ifdef __GNUC__ 261104477Ssam bus_dma_segment_t dm_segments[dmat->nsegments]; 262104477Ssam#else 263104477Ssam bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 264104477Ssam#endif 265104477Ssam bus_dma_segment_t *sg; 266104477Ssam int seg; 267104477Ssam int error = 0; 268104477Ssam vm_offset_t nextpaddr; 269104477Ssam 270104477Ssam if (map != NULL) 271104477Ssam panic("bus_dmamap_load: Invalid map\n"); 272104477Ssam 273104477Ssam vaddr = (vm_offset_t)buf; 274104477Ssam sg = &dm_segments[0]; 275104477Ssam seg = 1; 276104477Ssam sg->ds_len = 0; 277104477Ssam nextpaddr = 0; 278104477Ssam 279104477Ssam do { 280104477Ssam bus_size_t size; 281104477Ssam 282104477Ssam paddr = pmap_kextract(vaddr); 283104477Ssam size = PAGE_SIZE - (paddr & PAGE_MASK); 284104477Ssam if (size > buflen) 285104477Ssam size = buflen; 286104477Ssam 287104477Ssam if (sg->ds_len == 0) { 288104477Ssam sg->ds_addr = paddr; 289104477Ssam sg->ds_len = size; 290104477Ssam } else if (paddr == nextpaddr) { 291104477Ssam sg->ds_len += size; 292104477Ssam } else { 293104477Ssam /* Go to the next segment */ 294104477Ssam sg++; 295104477Ssam seg++; 296104477Ssam if (seg > dmat->nsegments) 297104477Ssam break; 298104477Ssam sg->ds_addr = paddr; 299104477Ssam sg->ds_len = size; 300104477Ssam } 301104477Ssam vaddr += size; 302104477Ssam nextpaddr = paddr + size; 303104477Ssam buflen -= size; 304104477Ssam 305104477Ssam } while (buflen > 0); 306104477Ssam 307104477Ssam if (buflen != 0) { 308104477Ssam printf("bus_dmamap_load: Too many segs! buf_len = 0x%lx\n", 309104477Ssam (u_long)buflen); 310104477Ssam error = EFBIG; 311104477Ssam } 312104477Ssam 313104477Ssam (*callback)(callback_arg, dm_segments, seg, error); 314104477Ssam 315104477Ssam return (0); 316104477Ssam} 317104477Ssam 318104477Ssam/* 319104477Ssam * Utility function to load a linear buffer. lastaddrp holds state 320104477Ssam * between invocations (for multiple-buffer loads). segp contains 321104477Ssam * the starting segment on entrance, and the ending segment on exit. 322104477Ssam * first indicates if this is the first invocation of this function. 323104477Ssam */ 324104477Ssamstatic int 325104477Ssambus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t segs[], 326104477Ssam void *buf, bus_size_t buflen, struct thread *td, 327104477Ssam int flags, vm_offset_t *lastaddrp, int *segp, 328104477Ssam int first) 329104477Ssam{ 330104477Ssam bus_size_t sgsize; 331104477Ssam bus_addr_t curaddr, lastaddr, baddr, bmask; 332104477Ssam vm_offset_t vaddr = (vm_offset_t)buf; 333104477Ssam int seg; 334104477Ssam pmap_t pmap; 335104477Ssam 336104477Ssam if (td != NULL) 337104477Ssam pmap = vmspace_pmap(td->td_proc->p_vmspace); 338104477Ssam else 339104477Ssam pmap = NULL; 340104477Ssam 341104477Ssam lastaddr = *lastaddrp; 342104477Ssam bmask = ~(dmat->boundary - 1); 343104477Ssam 344104477Ssam for (seg = *segp; buflen > 0 ; ) { 345104477Ssam /* 346104477Ssam * Get the physical address for this segment. 347104477Ssam */ 348104477Ssam if (pmap) 349104477Ssam curaddr = pmap_extract(pmap, vaddr); 350104477Ssam else 351104477Ssam curaddr = pmap_kextract(vaddr); 352104477Ssam 353104477Ssam /* 354104477Ssam * Compute the segment size, and adjust counts. 355104477Ssam */ 356104477Ssam sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 357104477Ssam if (buflen < sgsize) 358104477Ssam sgsize = buflen; 359104477Ssam 360104477Ssam /* 361104477Ssam * Make sure we don't cross any boundaries. 362104477Ssam */ 363104477Ssam if (dmat->boundary > 0) { 364104477Ssam baddr = (curaddr + dmat->boundary) & bmask; 365104477Ssam if (sgsize > (baddr - curaddr)) 366104477Ssam sgsize = (baddr - curaddr); 367104477Ssam } 368104477Ssam 369104477Ssam /* 370104477Ssam * Insert chunk into a segment, coalescing with 371104477Ssam * the previous segment if possible. 372104477Ssam */ 373104477Ssam if (first) { 374104477Ssam segs[seg].ds_addr = curaddr; 375104477Ssam segs[seg].ds_len = sgsize; 376104477Ssam first = 0; 377104477Ssam } else { 378104477Ssam if (curaddr == lastaddr && 379104477Ssam (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 380104477Ssam (dmat->boundary == 0 || 381104477Ssam (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 382104477Ssam segs[seg].ds_len += sgsize; 383104477Ssam else { 384104477Ssam if (++seg >= dmat->nsegments) 385104477Ssam break; 386104477Ssam segs[seg].ds_addr = curaddr; 387104477Ssam segs[seg].ds_len = sgsize; 388104477Ssam } 389104477Ssam } 390104477Ssam 391104477Ssam lastaddr = curaddr + sgsize; 392104477Ssam vaddr += sgsize; 393104477Ssam buflen -= sgsize; 394104477Ssam } 395104477Ssam 396104477Ssam *segp = seg; 397104477Ssam *lastaddrp = lastaddr; 398104477Ssam 399104477Ssam /* 400104477Ssam * Did we fit? 401104477Ssam */ 402104477Ssam return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 403104477Ssam} 404104477Ssam 405104477Ssam/* 406104477Ssam * Like bus_dmamap_load(), but for mbufs. 407104477Ssam */ 408104477Ssamint 409104477Ssambus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 410104477Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 411104477Ssam int flags) 412104477Ssam{ 413104477Ssam#ifdef __GNUC__ 414104477Ssam bus_dma_segment_t dm_segments[dmat->nsegments]; 415104477Ssam#else 416104477Ssam bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 417104477Ssam#endif 418104477Ssam int nsegs = 0, error = 0; 419104477Ssam 420104477Ssam M_ASSERTPKTHDR(m0); 421104477Ssam 422104477Ssam if (m0->m_pkthdr.len <= dmat->maxsize) { 423104477Ssam int first = 1; 424104477Ssam vm_offset_t lastaddr = 0; 425104477Ssam struct mbuf *m; 426104477Ssam 427104477Ssam for (m = m0; m != NULL && error == 0; m = m->m_next) { 428104477Ssam if (m->m_len > 0) { 429104477Ssam error = bus_dmamap_load_buffer(dmat, 430104477Ssam dm_segments, m->m_data, m->m_len, NULL, 431104477Ssam flags, &lastaddr, &nsegs, first); 432104477Ssam first = 0; 433104477Ssam } 434104477Ssam } 435104477Ssam } else { 436104477Ssam error = EINVAL; 437104477Ssam } 438104477Ssam 439104477Ssam if (error) { 440104477Ssam /* 441104477Ssam * force "no valid mappings" on error in callback. 442104477Ssam */ 443104477Ssam (*callback)(callback_arg, dm_segments, 0, 0, error); 444104477Ssam } else { 445104477Ssam (*callback)(callback_arg, dm_segments, nsegs+1, 446104477Ssam m0->m_pkthdr.len, error); 447104477Ssam } 448104477Ssam return (error); 449104477Ssam} 450104477Ssam 451104477Ssam/* 452104477Ssam * Like bus_dmamap_load(), but for uios. 453104477Ssam */ 454104477Ssamint 455104477Ssambus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 456104477Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 457104477Ssam int flags) 458104477Ssam{ 459104477Ssam vm_offset_t lastaddr; 460104477Ssam#ifdef __GNUC__ 461104477Ssam bus_dma_segment_t dm_segments[dmat->nsegments]; 462104477Ssam#else 463104477Ssam bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 464104477Ssam#endif 465104477Ssam int nsegs, i, error, first; 466104477Ssam bus_size_t resid; 467104477Ssam struct iovec *iov; 468104477Ssam struct proc *td = NULL; 469104477Ssam 470104477Ssam resid = uio->uio_resid; 471104477Ssam iov = uio->uio_iov; 472104477Ssam 473104477Ssam if (uio->uio_segflg == UIO_USERSPACE) { 474104918Ssam td = uio->uio_td; 475104918Ssam KASSERT(td != NULL, 476104477Ssam ("bus_dmamap_load_uio: USERSPACE but no proc")); 477104477Ssam } 478104477Ssam 479104477Ssam first = 1; 480104477Ssam nsegs = error = 0; 481104477Ssam for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 482104477Ssam /* 483104477Ssam * Now at the first iovec to load. Load each iovec 484104477Ssam * until we have exhausted the residual count. 485104477Ssam */ 486104477Ssam bus_size_t minlen = 487104477Ssam resid < iov[i].iov_len ? resid : iov[i].iov_len; 488104477Ssam caddr_t addr = (caddr_t) iov[i].iov_base; 489104477Ssam 490104477Ssam if (minlen > 0) { 491104477Ssam error = bus_dmamap_load_buffer(dmat, dm_segments, addr, 492104477Ssam minlen, td, flags, &lastaddr, &nsegs, first); 493104477Ssam 494104477Ssam first = 0; 495104477Ssam 496104477Ssam resid -= minlen; 497104477Ssam } 498104477Ssam } 499104477Ssam 500104477Ssam if (error) { 501104477Ssam /* 502104477Ssam * force "no valid mappings" on error in callback. 503104477Ssam */ 504104477Ssam (*callback)(callback_arg, dm_segments, 0, 0, error); 505104477Ssam } else { 506104477Ssam (*callback)(callback_arg, dm_segments, nsegs+1, 507104477Ssam uio->uio_resid, error); 508104477Ssam } 509104477Ssam 510104477Ssam return (error); 511104477Ssam} 512104477Ssam 513104477Ssam/* 514104477Ssam * Release the mapping held by map. A no-op on PowerPC. 515104477Ssam */ 516104477Ssamvoid 517104477Ssambus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 518104477Ssam{ 519104477Ssam 520104477Ssam return; 521104477Ssam} 522104477Ssam 523104477Ssamvoid 524104477Ssambus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, int op) 525104477Ssam{ 526104477Ssam 527104477Ssam return; 528104477Ssam} 529104477Ssam