busdma_machdep.c revision 118246
132516Sgibbs/* 240029Sgibbs * Copyright (c) 1997, 1998 Justin T. Gibbs. 332516Sgibbs * All rights reserved. 432516Sgibbs * 532516Sgibbs * Redistribution and use in source and binary forms, with or without 632516Sgibbs * modification, are permitted provided that the following conditions 732516Sgibbs * are met: 832516Sgibbs * 1. Redistributions of source code must retain the above copyright 932516Sgibbs * notice, this list of conditions, and the following disclaimer, 1032516Sgibbs * without modification, immediately at the beginning of the file. 1132516Sgibbs * 2. The name of the author may not be used to endorse or promote products 1232516Sgibbs * derived from this software without specific prior written permission. 1332516Sgibbs * 1432516Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1532516Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1632516Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1732516Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 1832516Sgibbs * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1932516Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2032516Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2132516Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2232516Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2332516Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2432516Sgibbs * SUCH DAMAGE. 2532516Sgibbs */ 2632516Sgibbs 27115683Sobrien#include <sys/cdefs.h> 28115683Sobrien__FBSDID("$FreeBSD: head/sys/i386/i386/busdma_machdep.c 118246 2003-07-31 05:34:20Z scottl $"); 29115683Sobrien 3032516Sgibbs#include <sys/param.h> 3132516Sgibbs#include <sys/systm.h> 3232516Sgibbs#include <sys/malloc.h> 3367551Sjhb#include <sys/bus.h> 3467551Sjhb#include <sys/interrupt.h> 35112346Smux#include <sys/kernel.h> 3676827Salfred#include <sys/lock.h> 3779224Sdillon#include <sys/proc.h> 3876827Salfred#include <sys/mutex.h> 39104486Ssam#include <sys/mbuf.h> 40104486Ssam#include <sys/uio.h> 4132516Sgibbs 4232516Sgibbs#include <vm/vm.h> 4332516Sgibbs#include <vm/vm_page.h> 44104486Ssam#include <vm/vm_map.h> 4532516Sgibbs 46112436Smux#include <machine/atomic.h> 4732516Sgibbs#include <machine/bus.h> 4832516Sgibbs#include <machine/md_var.h> 4932516Sgibbs 50113228Sjake#define MAX_BPAGES 512 5132516Sgibbs 5232516Sgibbsstruct bus_dma_tag { 5332516Sgibbs bus_dma_tag_t parent; 5435767Sgibbs bus_size_t alignment; 5532516Sgibbs bus_size_t boundary; 5632516Sgibbs bus_addr_t lowaddr; 5732516Sgibbs bus_addr_t highaddr; 5832516Sgibbs bus_dma_filter_t *filter; 5932516Sgibbs void *filterarg; 6032516Sgibbs bus_size_t maxsize; 6135767Sgibbs u_int nsegments; 6232516Sgibbs bus_size_t maxsegsz; 6332516Sgibbs int flags; 6432516Sgibbs int ref_count; 6532516Sgibbs int map_count; 66117126Sscottl bus_dma_lock_t *lockfunc; 67117126Sscottl void *lockfuncarg; 68118246Sscottl bus_dma_segment_t *segments; 6932516Sgibbs}; 7032516Sgibbs 7132516Sgibbsstruct bounce_page { 7232516Sgibbs vm_offset_t vaddr; /* kva of bounce buffer */ 7332516Sgibbs bus_addr_t busaddr; /* Physical address */ 7432516Sgibbs vm_offset_t datavaddr; /* kva of client data */ 7532516Sgibbs bus_size_t datacount; /* client data count */ 7660938Sjake STAILQ_ENTRY(bounce_page) links; 7732516Sgibbs}; 7832516Sgibbs 7932516Sgibbsint busdma_swi_pending; 8032516Sgibbs 81117136Smuxstatic struct mtx bounce_lock; 8260938Sjakestatic STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 8332516Sgibbsstatic int free_bpages; 8432516Sgibbsstatic int reserved_bpages; 8532516Sgibbsstatic int active_bpages; 8632516Sgibbsstatic int total_bpages; 8732516Sgibbsstatic bus_addr_t bounce_lowaddr = BUS_SPACE_MAXADDR; 8832516Sgibbs 8932516Sgibbsstruct bus_dmamap { 9032516Sgibbs struct bp_list bpages; 9132516Sgibbs int pagesneeded; 9232516Sgibbs int pagesreserved; 9332516Sgibbs bus_dma_tag_t dmat; 9432516Sgibbs void *buf; /* unmapped buffer pointer */ 9532516Sgibbs bus_size_t buflen; /* unmapped buffer length */ 9632516Sgibbs bus_dmamap_callback_t *callback; 9732516Sgibbs void *callback_arg; 9860938Sjake STAILQ_ENTRY(bus_dmamap) links; 9932516Sgibbs}; 10032516Sgibbs 10160938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 10260938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 10332516Sgibbsstatic struct bus_dmamap nobounce_dmamap; 10432516Sgibbs 105112346Smuxstatic void init_bounce_pages(void *dummy); 10632516Sgibbsstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 107113228Sjakestatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 108117136Smux int commit); 109112569Sjakestatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 11032516Sgibbs vm_offset_t vaddr, bus_size_t size); 11132516Sgibbsstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 11232516Sgibbsstatic __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 11332516Sgibbs 11495076Salfred/* 11595076Salfred * Return true if a match is made. 116117136Smux * 11795076Salfred * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 118117136Smux * 11995076Salfred * If paddr is within the bounds of the dma tag then call the filter callback 12095076Salfred * to check for a match, if there is no filter callback then assume a match. 12195076Salfred */ 12232516Sgibbsstatic __inline int 12332516Sgibbsrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 12432516Sgibbs{ 12532516Sgibbs int retval; 12632516Sgibbs 12732516Sgibbs retval = 0; 12832516Sgibbs do { 12932516Sgibbs if (paddr > dmat->lowaddr 13032516Sgibbs && paddr <= dmat->highaddr 13132516Sgibbs && (dmat->filter == NULL 13232516Sgibbs || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 13332516Sgibbs retval = 1; 13432516Sgibbs 13532516Sgibbs dmat = dmat->parent; 13632516Sgibbs } while (retval == 0 && dmat != NULL); 13732516Sgibbs return (retval); 13832516Sgibbs} 13932516Sgibbs 140117126Sscottl/* 141117126Sscottl * Convenience function for manipulating driver locks from busdma (during 142117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 143117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 144117126Sscottl * non-mutex locking scheme don't have to use this at all. 145117126Sscottl */ 146117126Sscottlvoid 147117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 148117126Sscottl{ 149117126Sscottl struct mtx *dmtx; 150117126Sscottl 151117126Sscottl dmtx = (struct mtx *)arg; 152117126Sscottl switch (op) { 153117126Sscottl case BUS_DMA_LOCK: 154117126Sscottl mtx_lock(dmtx); 155117126Sscottl break; 156117126Sscottl case BUS_DMA_UNLOCK: 157117126Sscottl mtx_unlock(dmtx); 158117126Sscottl break; 159117126Sscottl default: 160117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 161117126Sscottl } 162117126Sscottl} 163117126Sscottl 164117126Sscottl/* 165117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 166117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 167117126Sscottl * with the tag are meant to never be defered. 168117126Sscottl * XXX Should have a way to identify which driver is responsible here. 169117126Sscottl */ 170117126Sscottlstatic void 171117126Sscottldflt_lock(void *arg, bus_dma_lock_op_t op) 172117126Sscottl{ 173117126Sscottl panic("driver error: busdma dflt_lock called"); 174117126Sscottl} 175117126Sscottl 17635767Sgibbs#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 17732516Sgibbs/* 17832516Sgibbs * Allocate a device specific dma_tag. 17932516Sgibbs */ 18032516Sgibbsint 18135767Sgibbsbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 18235767Sgibbs bus_size_t boundary, bus_addr_t lowaddr, 18335767Sgibbs bus_addr_t highaddr, bus_dma_filter_t *filter, 18435767Sgibbs void *filterarg, bus_size_t maxsize, int nsegments, 185117126Sscottl bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 186117126Sscottl void *lockfuncarg, bus_dma_tag_t *dmat) 18732516Sgibbs{ 18832516Sgibbs bus_dma_tag_t newtag; 18932516Sgibbs int error = 0; 19032516Sgibbs 19132516Sgibbs /* Return a NULL tag on failure */ 19232516Sgibbs *dmat = NULL; 19332516Sgibbs 19432516Sgibbs newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 19532516Sgibbs if (newtag == NULL) 19632516Sgibbs return (ENOMEM); 19732516Sgibbs 19832516Sgibbs newtag->parent = parent; 19948449Smjacob newtag->alignment = alignment; 20032516Sgibbs newtag->boundary = boundary; 201112569Sjake newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 202112569Sjake newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 203112569Sjake (PAGE_SIZE - 1); 20432516Sgibbs newtag->filter = filter; 20532516Sgibbs newtag->filterarg = filterarg; 20632516Sgibbs newtag->maxsize = maxsize; 20732516Sgibbs newtag->nsegments = nsegments; 20832516Sgibbs newtag->maxsegsz = maxsegsz; 20932516Sgibbs newtag->flags = flags; 21032516Sgibbs newtag->ref_count = 1; /* Count ourself */ 21132516Sgibbs newtag->map_count = 0; 212117126Sscottl if (lockfunc != NULL) { 213117126Sscottl newtag->lockfunc = lockfunc; 214117126Sscottl newtag->lockfuncarg = lockfuncarg; 215117126Sscottl } else { 216117126Sscottl newtag->lockfunc = dflt_lock; 217117126Sscottl newtag->lockfuncarg = NULL; 218117126Sscottl } 219118246Sscottl newtag->segments = NULL; 220118246Sscottl 22132516Sgibbs /* Take into account any restrictions imposed by our parent tag */ 22232516Sgibbs if (parent != NULL) { 22332516Sgibbs newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 22432516Sgibbs newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 22532516Sgibbs /* 22632516Sgibbs * XXX Not really correct??? Probably need to honor boundary 22732516Sgibbs * all the way up the inheritence chain. 22832516Sgibbs */ 22935767Sgibbs newtag->boundary = MAX(parent->boundary, newtag->boundary); 23032516Sgibbs if (newtag->filter == NULL) { 23132516Sgibbs /* 23232516Sgibbs * Short circuit looking at our parent directly 23335256Sdes * since we have encapsulated all of its information 23432516Sgibbs */ 23532516Sgibbs newtag->filter = parent->filter; 23632516Sgibbs newtag->filterarg = parent->filterarg; 23732516Sgibbs newtag->parent = parent->parent; 23832516Sgibbs } 239112436Smux if (newtag->parent != NULL) 240112436Smux atomic_add_int(&parent->ref_count, 1); 24132516Sgibbs } 24232516Sgibbs 243112569Sjake if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) && 244112569Sjake (flags & BUS_DMA_ALLOCNOW) != 0) { 24532516Sgibbs /* Must bounce */ 24632516Sgibbs 24732516Sgibbs if (lowaddr > bounce_lowaddr) { 24832516Sgibbs /* 24932516Sgibbs * Go through the pool and kill any pages 25032516Sgibbs * that don't reside below lowaddr. 25132516Sgibbs */ 25235767Sgibbs panic("bus_dma_tag_create: page reallocation " 25332516Sgibbs "not implemented"); 25432516Sgibbs } 25532516Sgibbs if (ptoa(total_bpages) < maxsize) { 25632516Sgibbs int pages; 25732516Sgibbs 25832516Sgibbs pages = atop(maxsize) - total_bpages; 25932516Sgibbs 26032516Sgibbs /* Add pages to our bounce pool */ 26132516Sgibbs if (alloc_bounce_pages(newtag, pages) < pages) 26232516Sgibbs error = ENOMEM; 26332516Sgibbs } 26435767Sgibbs /* Performed initial allocation */ 26535767Sgibbs newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 26632516Sgibbs } 26732516Sgibbs 26832516Sgibbs if (error != 0) { 26932516Sgibbs free(newtag, M_DEVBUF); 27032516Sgibbs } else { 27132516Sgibbs *dmat = newtag; 27232516Sgibbs } 27332516Sgibbs return (error); 27432516Sgibbs} 27532516Sgibbs 27632516Sgibbsint 27732516Sgibbsbus_dma_tag_destroy(bus_dma_tag_t dmat) 27832516Sgibbs{ 27932516Sgibbs if (dmat != NULL) { 28032516Sgibbs 28132516Sgibbs if (dmat->map_count != 0) 28232516Sgibbs return (EBUSY); 28332516Sgibbs 28432516Sgibbs while (dmat != NULL) { 28532516Sgibbs bus_dma_tag_t parent; 28632516Sgibbs 28732516Sgibbs parent = dmat->parent; 288112436Smux atomic_subtract_int(&dmat->ref_count, 1); 28932516Sgibbs if (dmat->ref_count == 0) { 290118246Sscottl if (dmat->segments != NULL) 291118246Sscottl free(dmat->segments, M_DEVBUF); 29232516Sgibbs free(dmat, M_DEVBUF); 29340029Sgibbs /* 29440029Sgibbs * Last reference count, so 29540029Sgibbs * release our reference 29640029Sgibbs * count on our parent. 29740029Sgibbs */ 29840029Sgibbs dmat = parent; 29940029Sgibbs } else 30040029Sgibbs dmat = NULL; 30132516Sgibbs } 30232516Sgibbs } 30332516Sgibbs return (0); 30432516Sgibbs} 30532516Sgibbs 30632516Sgibbs/* 30732516Sgibbs * Allocate a handle for mapping from kva/uva/physical 30832516Sgibbs * address space into bus device space. 30932516Sgibbs */ 31032516Sgibbsint 31132516Sgibbsbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 31232516Sgibbs{ 31332516Sgibbs int error; 31432516Sgibbs 31532516Sgibbs error = 0; 31632516Sgibbs 317118246Sscottl if (dmat->segments == NULL) { 318118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 319118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 320118246Sscottl M_NOWAIT); 321118246Sscottl if (dmat->segments == NULL) 322118246Sscottl return (ENOMEM); 323118246Sscottl } 324118246Sscottl 325112569Sjake if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem)) { 32632516Sgibbs /* Must bounce */ 32732516Sgibbs int maxpages; 32832516Sgibbs 32932516Sgibbs *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 33069781Sdwmalone M_NOWAIT | M_ZERO); 33169781Sdwmalone if (*mapp == NULL) 33235767Sgibbs return (ENOMEM); 33369781Sdwmalone 33469781Sdwmalone /* Initialize the new map */ 33569781Sdwmalone STAILQ_INIT(&((*mapp)->bpages)); 33669781Sdwmalone 33732516Sgibbs /* 33832516Sgibbs * Attempt to add pages to our pool on a per-instance 33932516Sgibbs * basis up to a sane limit. 34032516Sgibbs */ 34132516Sgibbs maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->lowaddr)); 34235767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 34335767Sgibbs || (dmat->map_count > 0 34435767Sgibbs && total_bpages < maxpages)) { 34532516Sgibbs int pages; 34632516Sgibbs 34735767Sgibbs if (dmat->lowaddr > bounce_lowaddr) { 34835767Sgibbs /* 34935767Sgibbs * Go through the pool and kill any pages 35035767Sgibbs * that don't reside below lowaddr. 35135767Sgibbs */ 35235767Sgibbs panic("bus_dmamap_create: page reallocation " 35335767Sgibbs "not implemented"); 35435767Sgibbs } 355113228Sjake pages = MAX(atop(dmat->maxsize), 1); 35632516Sgibbs pages = MIN(maxpages - total_bpages, pages); 357113228Sjake if (alloc_bounce_pages(dmat, pages) < pages) 358113228Sjake error = ENOMEM; 35935767Sgibbs 36035767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 36135767Sgibbs if (error == 0) 36235767Sgibbs dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 36335767Sgibbs } else { 36435767Sgibbs error = 0; 36535767Sgibbs } 36632516Sgibbs } 36732516Sgibbs } else { 36840029Sgibbs *mapp = NULL; 36932516Sgibbs } 37032516Sgibbs if (error == 0) 37132516Sgibbs dmat->map_count++; 37232516Sgibbs return (error); 37332516Sgibbs} 37432516Sgibbs 37532516Sgibbs/* 37632516Sgibbs * Destroy a handle for mapping from kva/uva/physical 37732516Sgibbs * address space into bus device space. 37832516Sgibbs */ 37932516Sgibbsint 38032516Sgibbsbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 38132516Sgibbs{ 382117136Smux if (map != NULL && map != &nobounce_dmamap) { 38332516Sgibbs if (STAILQ_FIRST(&map->bpages) != NULL) 38432516Sgibbs return (EBUSY); 38532516Sgibbs free(map, M_DEVBUF); 38632516Sgibbs } 38732516Sgibbs dmat->map_count--; 38832516Sgibbs return (0); 38932516Sgibbs} 39032516Sgibbs 39135767Sgibbs 39235767Sgibbs/* 39335767Sgibbs * Allocate a piece of memory that can be efficiently mapped into 39435767Sgibbs * bus device space based on the constraints lited in the dma tag. 39535767Sgibbs * A dmamap to for use with dmamap_load is also allocated. 39635767Sgibbs */ 39735767Sgibbsint 398115316Sscottlbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 399115316Sscottl bus_dmamap_t *mapp) 40035767Sgibbs{ 401118081Smux int mflags; 402118081Smux 403118081Smux if (flags & BUS_DMA_NOWAIT) 404118081Smux mflags = M_NOWAIT; 405118081Smux else 406118081Smux mflags = M_WAITOK; 407118081Smux if (flags & BUS_DMA_ZERO) 408118081Smux mflags |= M_ZERO; 409118081Smux 41035767Sgibbs /* If we succeed, no mapping/bouncing will be required */ 41140029Sgibbs *mapp = NULL; 41235767Sgibbs 413118246Sscottl if (dmat->segments == NULL) { 414118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 415118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 416118246Sscottl M_NOWAIT); 417118246Sscottl if (dmat->segments == NULL) 418118246Sscottl return (ENOMEM); 419118246Sscottl } 420118246Sscottl 421115316Sscottl if ((dmat->maxsize <= PAGE_SIZE) && 422112569Sjake dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 423118081Smux *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 42435767Sgibbs } else { 42535767Sgibbs /* 42635767Sgibbs * XXX Use Contigmalloc until it is merged into this facility 42735767Sgibbs * and handles multi-seg allocations. Nobody is doing 42835767Sgibbs * multi-seg allocations yet though. 42935767Sgibbs */ 430118081Smux *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 43148449Smjacob 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 43248449Smjacob dmat->boundary); 43335767Sgibbs } 43435767Sgibbs if (*vaddr == NULL) 43535767Sgibbs return (ENOMEM); 43635767Sgibbs return (0); 43735767Sgibbs} 43835767Sgibbs 43935767Sgibbs/* 44035767Sgibbs * Free a piece of memory and it's allociated dmamap, that was allocated 44195076Salfred * via bus_dmamem_alloc. Make the same choice for free/contigfree. 44235767Sgibbs */ 44335767Sgibbsvoid 444115316Sscottlbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 44535767Sgibbs{ 44635767Sgibbs /* 44735767Sgibbs * dmamem does not need to be bounced, so the map should be 44835767Sgibbs * NULL 44935767Sgibbs */ 45049859Sgibbs if (map != NULL) 45135767Sgibbs panic("bus_dmamem_free: Invalid map freed\n"); 452115316Sscottl if ((dmat->maxsize <= PAGE_SIZE) 453115316Sscottl && dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 45440029Sgibbs free(vaddr, M_DEVBUF); 455112196Smux else { 456112196Smux mtx_lock(&Giant); 457115316Sscottl contigfree(vaddr, dmat->maxsize, M_DEVBUF); 458112196Smux mtx_unlock(&Giant); 459112196Smux } 46035767Sgibbs} 46135767Sgibbs 46232516Sgibbs/* 463104486Ssam * Utility function to load a linear buffer. lastaddrp holds state 464104486Ssam * between invocations (for multiple-buffer loads). segp contains 465104486Ssam * the starting segment on entrace, and the ending segment on exit. 466104486Ssam * first indicates if this is the first invocation of this function. 467104486Ssam */ 468104486Ssamstatic int 469104486Ssam_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 470113228Sjake bus_dmamap_t map, 471104486Ssam void *buf, bus_size_t buflen, 472104486Ssam struct thread *td, 473104486Ssam int flags, 474113228Sjake bus_addr_t *lastaddrp, 475104486Ssam int *segp, 476104486Ssam int first) 477104486Ssam{ 478118246Sscottl bus_dma_segment_t *segs; 479104486Ssam bus_size_t sgsize; 480104486Ssam bus_addr_t curaddr, lastaddr, baddr, bmask; 481113228Sjake vm_offset_t vaddr; 482113228Sjake bus_addr_t paddr; 483113228Sjake int needbounce = 0; 484104486Ssam int seg; 485104486Ssam pmap_t pmap; 486104486Ssam 487118246Sscottl segs = dmat->segments; 488118246Sscottl 489113228Sjake if (map == NULL) 490113228Sjake map = &nobounce_dmamap; 491113228Sjake 492104486Ssam if (td != NULL) 493104486Ssam pmap = vmspace_pmap(td->td_proc->p_vmspace); 494104486Ssam else 495104486Ssam pmap = NULL; 496104486Ssam 497113228Sjake if (dmat->lowaddr < ptoa((vm_paddr_t)Maxmem)) { 498113228Sjake vm_offset_t vendaddr; 499113228Sjake 500113228Sjake /* 501113228Sjake * Count the number of bounce pages 502113228Sjake * needed in order to complete this transfer 503113228Sjake */ 504113228Sjake vaddr = trunc_page((vm_offset_t)buf); 505113228Sjake vendaddr = (vm_offset_t)buf + buflen; 506113228Sjake 507113228Sjake while (vaddr < vendaddr) { 508113228Sjake paddr = pmap_kextract(vaddr); 509113228Sjake if (run_filter(dmat, paddr) != 0) { 510113228Sjake needbounce = 1; 511113228Sjake map->pagesneeded++; 512113228Sjake } 513113228Sjake vaddr += PAGE_SIZE; 514113228Sjake } 515113228Sjake } 516113228Sjake 517113228Sjake vaddr = (vm_offset_t)buf; 518113228Sjake 519113228Sjake /* Reserve Necessary Bounce Pages */ 520113228Sjake if (map->pagesneeded != 0) { 521113228Sjake mtx_lock(&bounce_lock); 522113472Ssimokawa if (flags & BUS_DMA_NOWAIT) { 523113472Ssimokawa if (reserve_bounce_pages(dmat, map, 0) != 0) { 524113472Ssimokawa mtx_unlock(&bounce_lock); 525113472Ssimokawa return (ENOMEM); 526113472Ssimokawa } 527113472Ssimokawa } else { 528113472Ssimokawa if (reserve_bounce_pages(dmat, map, 1) != 0) { 529113472Ssimokawa /* Queue us for resources */ 530113472Ssimokawa map->dmat = dmat; 531113472Ssimokawa map->buf = buf; 532113472Ssimokawa map->buflen = buflen; 533113472Ssimokawa STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 534117136Smux map, links); 535113472Ssimokawa mtx_unlock(&bounce_lock); 536113472Ssimokawa return (EINPROGRESS); 537113472Ssimokawa } 538113228Sjake } 539113228Sjake mtx_unlock(&bounce_lock); 540113228Sjake } 541113228Sjake 542104486Ssam lastaddr = *lastaddrp; 543113228Sjake bmask = ~(dmat->boundary - 1); 544104486Ssam 545104486Ssam for (seg = *segp; buflen > 0 ; ) { 546104486Ssam /* 547104486Ssam * Get the physical address for this segment. 548104486Ssam */ 549104486Ssam if (pmap) 550104486Ssam curaddr = pmap_extract(pmap, vaddr); 551104486Ssam else 552104486Ssam curaddr = pmap_kextract(vaddr); 553104486Ssam 554104486Ssam /* 555104486Ssam * Compute the segment size, and adjust counts. 556104486Ssam */ 557104486Ssam sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 558104486Ssam if (buflen < sgsize) 559104486Ssam sgsize = buflen; 560104486Ssam 561104486Ssam /* 562104486Ssam * Make sure we don't cross any boundaries. 563104486Ssam */ 564104486Ssam if (dmat->boundary > 0) { 565104486Ssam baddr = (curaddr + dmat->boundary) & bmask; 566104486Ssam if (sgsize > (baddr - curaddr)) 567104486Ssam sgsize = (baddr - curaddr); 568104486Ssam } 569104486Ssam 570113228Sjake if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) 571113228Sjake curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 572113228Sjake 573104486Ssam /* 574104486Ssam * Insert chunk into a segment, coalescing with 575104486Ssam * previous segment if possible. 576104486Ssam */ 577104486Ssam if (first) { 578104486Ssam segs[seg].ds_addr = curaddr; 579104486Ssam segs[seg].ds_len = sgsize; 580104486Ssam first = 0; 581104486Ssam } else { 582113228Sjake if (needbounce == 0 && curaddr == lastaddr && 583104486Ssam (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 584104486Ssam (dmat->boundary == 0 || 585104486Ssam (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 586104486Ssam segs[seg].ds_len += sgsize; 587104486Ssam else { 588104486Ssam if (++seg >= dmat->nsegments) 589104486Ssam break; 590104486Ssam segs[seg].ds_addr = curaddr; 591104486Ssam segs[seg].ds_len = sgsize; 592104486Ssam } 593104486Ssam } 594104486Ssam 595104486Ssam lastaddr = curaddr + sgsize; 596104486Ssam vaddr += sgsize; 597104486Ssam buflen -= sgsize; 598104486Ssam } 599104486Ssam 600104486Ssam *segp = seg; 601104486Ssam *lastaddrp = lastaddr; 602104486Ssam 603104486Ssam /* 604104486Ssam * Did we fit? 605104486Ssam */ 606104486Ssam return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 607104486Ssam} 608104486Ssam 609113459Ssimokawa#define BUS_DMAMAP_NSEGS ((64 * 1024) / PAGE_SIZE + 1) 610113459Ssimokawa 611104486Ssam/* 612113459Ssimokawa * Map the buffer buf into bus space using the dmamap map. 613113459Ssimokawa */ 614113459Ssimokawaint 615113459Ssimokawabus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 616113459Ssimokawa bus_size_t buflen, bus_dmamap_callback_t *callback, 617113459Ssimokawa void *callback_arg, int flags) 618113459Ssimokawa{ 619113492Smux bus_addr_t lastaddr = 0; 620113459Ssimokawa int error, nsegs = 0; 621113459Ssimokawa 622113472Ssimokawa if (map != NULL) { 623113472Ssimokawa flags |= BUS_DMA_WAITOK; 624113472Ssimokawa map->callback = callback; 625113472Ssimokawa map->callback_arg = callback_arg; 626113472Ssimokawa } 627113472Ssimokawa 628118246Sscottl error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 629118246Sscottl &lastaddr, &nsegs, 1); 630113459Ssimokawa 631113472Ssimokawa if (error == EINPROGRESS) 632113492Smux return (error); 633113472Ssimokawa 634113459Ssimokawa if (error) 635118246Sscottl (*callback)(callback_arg, dmat->segments, 0, error); 636113459Ssimokawa else 637118246Sscottl (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 638113459Ssimokawa 639113459Ssimokawa return (0); 640113459Ssimokawa} 641113459Ssimokawa 642113459Ssimokawa 643113459Ssimokawa/* 644104486Ssam * Like _bus_dmamap_load(), but for mbufs. 645104486Ssam */ 646104486Ssamint 647104486Ssambus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 648104486Ssam struct mbuf *m0, 649104486Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 650104486Ssam int flags) 651104486Ssam{ 652104486Ssam int nsegs, error; 653104486Ssam 654117136Smux M_ASSERTPKTHDR(m0); 655104486Ssam 656113472Ssimokawa flags |= BUS_DMA_NOWAIT; 657104486Ssam nsegs = 0; 658104486Ssam error = 0; 659104486Ssam if (m0->m_pkthdr.len <= dmat->maxsize) { 660104486Ssam int first = 1; 661113228Sjake bus_addr_t lastaddr = 0; 662104486Ssam struct mbuf *m; 663104486Ssam 664104486Ssam for (m = m0; m != NULL && error == 0; m = m->m_next) { 665110335Sharti if (m->m_len > 0) { 666113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 667110335Sharti m->m_data, m->m_len, 668110335Sharti NULL, flags, &lastaddr, 669110335Sharti &nsegs, first); 670110335Sharti first = 0; 671110335Sharti } 672104486Ssam } 673104486Ssam } else { 674104486Ssam error = EINVAL; 675104486Ssam } 676104486Ssam 677104486Ssam if (error) { 678104486Ssam /* force "no valid mappings" in callback */ 679118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 680104486Ssam } else { 681118246Sscottl (*callback)(callback_arg, dmat->segments, 682104486Ssam nsegs+1, m0->m_pkthdr.len, error); 683104486Ssam } 684104486Ssam return (error); 685104486Ssam} 686104486Ssam 687104486Ssam/* 688104486Ssam * Like _bus_dmamap_load(), but for uios. 689104486Ssam */ 690104486Ssamint 691104486Ssambus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 692104486Ssam struct uio *uio, 693104486Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 694104486Ssam int flags) 695104486Ssam{ 696113228Sjake bus_addr_t lastaddr; 697104486Ssam int nsegs, error, first, i; 698104486Ssam bus_size_t resid; 699104486Ssam struct iovec *iov; 700104486Ssam struct thread *td = NULL; 701104486Ssam 702113472Ssimokawa flags |= BUS_DMA_NOWAIT; 703104486Ssam resid = uio->uio_resid; 704104486Ssam iov = uio->uio_iov; 705104486Ssam 706104486Ssam if (uio->uio_segflg == UIO_USERSPACE) { 707104486Ssam td = uio->uio_td; 708104486Ssam KASSERT(td != NULL, 709104486Ssam ("bus_dmamap_load_uio: USERSPACE but no proc")); 710104486Ssam } 711104486Ssam 712104486Ssam nsegs = 0; 713104486Ssam error = 0; 714104486Ssam first = 1; 715104486Ssam for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 716104486Ssam /* 717104486Ssam * Now at the first iovec to load. Load each iovec 718104486Ssam * until we have exhausted the residual count. 719104486Ssam */ 720104486Ssam bus_size_t minlen = 721104486Ssam resid < iov[i].iov_len ? resid : iov[i].iov_len; 722104486Ssam caddr_t addr = (caddr_t) iov[i].iov_base; 723104486Ssam 724110335Sharti if (minlen > 0) { 725113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 726110335Sharti addr, minlen, 727110335Sharti td, flags, &lastaddr, &nsegs, first); 728110335Sharti first = 0; 729104486Ssam 730110335Sharti resid -= minlen; 731110335Sharti } 732104486Ssam } 733104486Ssam 734104486Ssam if (error) { 735104486Ssam /* force "no valid mappings" in callback */ 736118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 737104486Ssam } else { 738118246Sscottl (*callback)(callback_arg, dmat->segments, 739104486Ssam nsegs+1, uio->uio_resid, error); 740104486Ssam } 741104486Ssam return (error); 742104486Ssam} 743104486Ssam 744104486Ssam/* 74532516Sgibbs * Release the mapping held by map. 74632516Sgibbs */ 74732516Sgibbsvoid 74832516Sgibbs_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 74932516Sgibbs{ 75032516Sgibbs struct bounce_page *bpage; 75132516Sgibbs 75232516Sgibbs while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 75332516Sgibbs STAILQ_REMOVE_HEAD(&map->bpages, links); 75432516Sgibbs free_bounce_page(dmat, bpage); 75532516Sgibbs } 75632516Sgibbs} 75732516Sgibbs 75832516Sgibbsvoid 759115343Sscottl_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 76032516Sgibbs{ 76132516Sgibbs struct bounce_page *bpage; 76232516Sgibbs 76332516Sgibbs if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 76432516Sgibbs /* 76532516Sgibbs * Handle data bouncing. We might also 76632516Sgibbs * want to add support for invalidating 76732516Sgibbs * the caches on broken hardware 76832516Sgibbs */ 769113347Smux if (op & BUS_DMASYNC_PREWRITE) { 77032516Sgibbs while (bpage != NULL) { 77132516Sgibbs bcopy((void *)bpage->datavaddr, 77232516Sgibbs (void *)bpage->vaddr, 77332516Sgibbs bpage->datacount); 77432516Sgibbs bpage = STAILQ_NEXT(bpage, links); 77532516Sgibbs } 776113347Smux } 77732516Sgibbs 778113347Smux if (op & BUS_DMASYNC_POSTREAD) { 77932516Sgibbs while (bpage != NULL) { 78032516Sgibbs bcopy((void *)bpage->vaddr, 78132516Sgibbs (void *)bpage->datavaddr, 78232516Sgibbs bpage->datacount); 78332516Sgibbs bpage = STAILQ_NEXT(bpage, links); 78432516Sgibbs } 78532516Sgibbs } 78632516Sgibbs } 78732516Sgibbs} 78832516Sgibbs 789112346Smuxstatic void 790112346Smuxinit_bounce_pages(void *dummy __unused) 791112346Smux{ 792112346Smux 793112346Smux free_bpages = 0; 794112346Smux reserved_bpages = 0; 795112346Smux active_bpages = 0; 796112346Smux total_bpages = 0; 797112346Smux STAILQ_INIT(&bounce_page_list); 798112346Smux STAILQ_INIT(&bounce_map_waitinglist); 799112346Smux STAILQ_INIT(&bounce_map_callbacklist); 800112346Smux mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 801112346Smux} 802112346SmuxSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 803112346Smux 80432516Sgibbsstatic int 80532516Sgibbsalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 80632516Sgibbs{ 80732516Sgibbs int count; 80832516Sgibbs 80932516Sgibbs count = 0; 81032516Sgibbs while (numpages > 0) { 81132516Sgibbs struct bounce_page *bpage; 81232516Sgibbs 81332516Sgibbs bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 81469781Sdwmalone M_NOWAIT | M_ZERO); 81532516Sgibbs 81632516Sgibbs if (bpage == NULL) 81732516Sgibbs break; 81832516Sgibbs bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 81932516Sgibbs M_NOWAIT, 0ul, 82032516Sgibbs dmat->lowaddr, 82135767Sgibbs PAGE_SIZE, 822117129Smux dmat->boundary); 823102241Sarchie if (bpage->vaddr == 0) { 82432516Sgibbs free(bpage, M_DEVBUF); 82532516Sgibbs break; 82632516Sgibbs } 82732516Sgibbs bpage->busaddr = pmap_kextract(bpage->vaddr); 828112346Smux mtx_lock(&bounce_lock); 82932516Sgibbs STAILQ_INSERT_TAIL(&bounce_page_list, bpage, links); 83032516Sgibbs total_bpages++; 83132516Sgibbs free_bpages++; 832112346Smux mtx_unlock(&bounce_lock); 83332516Sgibbs count++; 83432516Sgibbs numpages--; 83532516Sgibbs } 83632516Sgibbs return (count); 83732516Sgibbs} 83832516Sgibbs 83932516Sgibbsstatic int 840113228Sjakereserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 84132516Sgibbs{ 84232516Sgibbs int pages; 84332516Sgibbs 844112346Smux mtx_assert(&bounce_lock, MA_OWNED); 84532516Sgibbs pages = MIN(free_bpages, map->pagesneeded - map->pagesreserved); 846113228Sjake if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 847113228Sjake return (map->pagesneeded - (map->pagesreserved + pages)); 84832516Sgibbs free_bpages -= pages; 84932516Sgibbs reserved_bpages += pages; 85032516Sgibbs map->pagesreserved += pages; 85132516Sgibbs pages = map->pagesneeded - map->pagesreserved; 85232516Sgibbs 85332516Sgibbs return (pages); 85432516Sgibbs} 85532516Sgibbs 856112569Sjakestatic bus_addr_t 85732516Sgibbsadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 85832516Sgibbs bus_size_t size) 85932516Sgibbs{ 86032516Sgibbs struct bounce_page *bpage; 86132516Sgibbs 862113228Sjake KASSERT(map != NULL && map != &nobounce_dmamap, 863113228Sjake ("add_bounce_page: bad map %p", map)); 864113228Sjake 86532516Sgibbs if (map->pagesneeded == 0) 86632516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 86732516Sgibbs map->pagesneeded--; 86832516Sgibbs 86932516Sgibbs if (map->pagesreserved == 0) 87032516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 87132516Sgibbs map->pagesreserved--; 87232516Sgibbs 873112346Smux mtx_lock(&bounce_lock); 87432516Sgibbs bpage = STAILQ_FIRST(&bounce_page_list); 87532516Sgibbs if (bpage == NULL) 87632516Sgibbs panic("add_bounce_page: free page list is empty"); 87732516Sgibbs 87832516Sgibbs STAILQ_REMOVE_HEAD(&bounce_page_list, links); 87932516Sgibbs reserved_bpages--; 88032516Sgibbs active_bpages++; 881112346Smux mtx_unlock(&bounce_lock); 88232516Sgibbs 88332516Sgibbs bpage->datavaddr = vaddr; 88432516Sgibbs bpage->datacount = size; 88532516Sgibbs STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 88632516Sgibbs return (bpage->busaddr); 88732516Sgibbs} 88832516Sgibbs 88932516Sgibbsstatic void 89032516Sgibbsfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 89132516Sgibbs{ 89232516Sgibbs struct bus_dmamap *map; 89332516Sgibbs 89432516Sgibbs bpage->datavaddr = 0; 89532516Sgibbs bpage->datacount = 0; 89632516Sgibbs 897112346Smux mtx_lock(&bounce_lock); 89832516Sgibbs STAILQ_INSERT_HEAD(&bounce_page_list, bpage, links); 89932516Sgibbs free_bpages++; 90032516Sgibbs active_bpages--; 90132516Sgibbs if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 902113228Sjake if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 90332516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 90432516Sgibbs STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 90532516Sgibbs map, links); 90632516Sgibbs busdma_swi_pending = 1; 90788900Sjhb swi_sched(vm_ih, 0); 90832516Sgibbs } 90932516Sgibbs } 910112346Smux mtx_unlock(&bounce_lock); 91132516Sgibbs} 91232516Sgibbs 91332516Sgibbsvoid 91495076Salfredbusdma_swi(void) 91532516Sgibbs{ 916117126Sscottl bus_dma_tag_t dmat; 91732516Sgibbs struct bus_dmamap *map; 91832516Sgibbs 919112346Smux mtx_lock(&bounce_lock); 92032516Sgibbs while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 92132516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 922112346Smux mtx_unlock(&bounce_lock); 923117136Smux dmat = map->dmat; 924117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 92532516Sgibbs bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 92632516Sgibbs map->callback, map->callback_arg, /*flags*/0); 927117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 928112346Smux mtx_lock(&bounce_lock); 92932516Sgibbs } 930112346Smux mtx_unlock(&bounce_lock); 93132516Sgibbs} 932