busdma_machdep.c revision 137965
132516Sgibbs/* 240029Sgibbs * Copyright (c) 1997, 1998 Justin T. Gibbs. 332516Sgibbs * All rights reserved. 432516Sgibbs * 532516Sgibbs * Redistribution and use in source and binary forms, with or without 632516Sgibbs * modification, are permitted provided that the following conditions 732516Sgibbs * are met: 832516Sgibbs * 1. Redistributions of source code must retain the above copyright 932516Sgibbs * notice, this list of conditions, and the following disclaimer, 1032516Sgibbs * without modification, immediately at the beginning of the file. 1132516Sgibbs * 2. The name of the author may not be used to endorse or promote products 1232516Sgibbs * derived from this software without specific prior written permission. 1332516Sgibbs * 1432516Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1532516Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1632516Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1732516Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 1832516Sgibbs * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1932516Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2032516Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2132516Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2232516Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2332516Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2432516Sgibbs * SUCH DAMAGE. 2532516Sgibbs */ 2632516Sgibbs 27115683Sobrien#include <sys/cdefs.h> 28115683Sobrien__FBSDID("$FreeBSD: head/sys/i386/i386/busdma_machdep.c 137965 2004-11-21 04:43:28Z scottl $"); 29115683Sobrien 3032516Sgibbs#include <sys/param.h> 3132516Sgibbs#include <sys/systm.h> 3232516Sgibbs#include <sys/malloc.h> 3367551Sjhb#include <sys/bus.h> 3467551Sjhb#include <sys/interrupt.h> 35112346Smux#include <sys/kernel.h> 36136805Srwatson#include <sys/ktr.h> 3776827Salfred#include <sys/lock.h> 3879224Sdillon#include <sys/proc.h> 3976827Salfred#include <sys/mutex.h> 40104486Ssam#include <sys/mbuf.h> 41104486Ssam#include <sys/uio.h> 42131529Sscottl#include <sys/sysctl.h> 43137142Sscottl#include <sys/ktr.h> 4432516Sgibbs 4532516Sgibbs#include <vm/vm.h> 4632516Sgibbs#include <vm/vm_page.h> 47104486Ssam#include <vm/vm_map.h> 4832516Sgibbs 49112436Smux#include <machine/atomic.h> 5032516Sgibbs#include <machine/bus.h> 5132516Sgibbs#include <machine/md_var.h> 5232516Sgibbs 53113228Sjake#define MAX_BPAGES 512 5432516Sgibbs 55137445Sscottlstruct bounce_zone; 56137445Sscottl 5732516Sgibbsstruct bus_dma_tag { 5832516Sgibbs bus_dma_tag_t parent; 5935767Sgibbs bus_size_t alignment; 6032516Sgibbs bus_size_t boundary; 6132516Sgibbs bus_addr_t lowaddr; 6232516Sgibbs bus_addr_t highaddr; 6332516Sgibbs bus_dma_filter_t *filter; 6432516Sgibbs void *filterarg; 6532516Sgibbs bus_size_t maxsize; 6635767Sgibbs u_int nsegments; 6732516Sgibbs bus_size_t maxsegsz; 6832516Sgibbs int flags; 6932516Sgibbs int ref_count; 7032516Sgibbs int map_count; 71117126Sscottl bus_dma_lock_t *lockfunc; 72117126Sscottl void *lockfuncarg; 73118246Sscottl bus_dma_segment_t *segments; 74137445Sscottl struct bounce_zone *bounce_zone; 7532516Sgibbs}; 7632516Sgibbs 77132545Sscottlstruct bounce_page { 78132545Sscottl vm_offset_t vaddr; /* kva of bounce buffer */ 79132545Sscottl bus_addr_t busaddr; /* Physical address */ 80132545Sscottl vm_offset_t datavaddr; /* kva of client data */ 81132545Sscottl bus_size_t datacount; /* client data count */ 82132545Sscottl STAILQ_ENTRY(bounce_page) links; 83132545Sscottl}; 84132545Sscottl 8532516Sgibbsint busdma_swi_pending; 8632516Sgibbs 87137445Sscottlstruct bounce_zone { 88137445Sscottl STAILQ_ENTRY(bounce_zone) links; 89137445Sscottl STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 90137965Sscottl int total_bpages; 91137445Sscottl int free_bpages; 92137445Sscottl int reserved_bpages; 93137445Sscottl int active_bpages; 94137445Sscottl int total_bounced; 95137445Sscottl int total_deferred; 96137445Sscottl bus_size_t alignment; 97137445Sscottl bus_size_t boundary; 98137445Sscottl bus_addr_t lowaddr; 99137445Sscottl char zoneid[8]; 100137445Sscottl char lowaddrid[20]; 101137445Sscottl struct sysctl_ctx_list sysctl_tree; 102137445Sscottl struct sysctl_oid *sysctl_tree_top; 103137445Sscottl}; 104137445Sscottl 105117136Smuxstatic struct mtx bounce_lock; 10632516Sgibbsstatic int total_bpages; 107137445Sscottlstatic int busdma_zonecount; 108137445Sscottlstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 10932516Sgibbsstatic bus_addr_t bounce_lowaddr = BUS_SPACE_MAXADDR; 11032516Sgibbs 111131529SscottlSYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 112131529SscottlSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 113131529Sscottl "Total bounce pages"); 114131529Sscottl 11532516Sgibbsstruct bus_dmamap { 11632516Sgibbs struct bp_list bpages; 11732516Sgibbs int pagesneeded; 11832516Sgibbs int pagesreserved; 11932516Sgibbs bus_dma_tag_t dmat; 12032516Sgibbs void *buf; /* unmapped buffer pointer */ 12132516Sgibbs bus_size_t buflen; /* unmapped buffer length */ 12232516Sgibbs bus_dmamap_callback_t *callback; 12332516Sgibbs void *callback_arg; 12460938Sjake STAILQ_ENTRY(bus_dmamap) links; 12532516Sgibbs}; 12632516Sgibbs 12760938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 12860938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 12932516Sgibbsstatic struct bus_dmamap nobounce_dmamap; 13032516Sgibbs 131112346Smuxstatic void init_bounce_pages(void *dummy); 132137965Sscottlstatic int alloc_bounce_zone(bus_dma_tag_t dmat); 13332516Sgibbsstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 134113228Sjakestatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 135117136Smux int commit); 136112569Sjakestatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 13732516Sgibbs vm_offset_t vaddr, bus_size_t size); 13832516Sgibbsstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 139137894Sscottlstatic __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 14032516Sgibbs 14195076Salfred/* 14295076Salfred * Return true if a match is made. 143117136Smux * 14495076Salfred * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 145117136Smux * 14695076Salfred * If paddr is within the bounds of the dma tag then call the filter callback 14795076Salfred * to check for a match, if there is no filter callback then assume a match. 14895076Salfred */ 14932516Sgibbsstatic __inline int 150137894Sscottlrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 15132516Sgibbs{ 15232516Sgibbs int retval; 15332516Sgibbs 15432516Sgibbs retval = 0; 155131529Sscottl 15632516Sgibbs do { 157131529Sscottl if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 158137894Sscottl || ((paddr & (dmat->alignment - 1)) != 0)) 15932516Sgibbs && (dmat->filter == NULL 160132545Sscottl || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 16132516Sgibbs retval = 1; 16232516Sgibbs 16332516Sgibbs dmat = dmat->parent; 16432516Sgibbs } while (retval == 0 && dmat != NULL); 16532516Sgibbs return (retval); 16632516Sgibbs} 16732516Sgibbs 168117126Sscottl/* 169117126Sscottl * Convenience function for manipulating driver locks from busdma (during 170117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 171117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 172117126Sscottl * non-mutex locking scheme don't have to use this at all. 173117126Sscottl */ 174117126Sscottlvoid 175117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 176117126Sscottl{ 177117126Sscottl struct mtx *dmtx; 178117126Sscottl 179117126Sscottl dmtx = (struct mtx *)arg; 180117126Sscottl switch (op) { 181117126Sscottl case BUS_DMA_LOCK: 182117126Sscottl mtx_lock(dmtx); 183117126Sscottl break; 184117126Sscottl case BUS_DMA_UNLOCK: 185117126Sscottl mtx_unlock(dmtx); 186117126Sscottl break; 187117126Sscottl default: 188117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 189117126Sscottl } 190117126Sscottl} 191117126Sscottl 192117126Sscottl/* 193117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 194117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 195117126Sscottl * with the tag are meant to never be defered. 196117126Sscottl * XXX Should have a way to identify which driver is responsible here. 197117126Sscottl */ 198117126Sscottlstatic void 199117126Sscottldflt_lock(void *arg, bus_dma_lock_op_t op) 200117126Sscottl{ 201117126Sscottl panic("driver error: busdma dflt_lock called"); 202117126Sscottl} 203117126Sscottl 204137965Sscottl#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 205137965Sscottl#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 20632516Sgibbs/* 20732516Sgibbs * Allocate a device specific dma_tag. 20832516Sgibbs */ 20932516Sgibbsint 21035767Sgibbsbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 21135767Sgibbs bus_size_t boundary, bus_addr_t lowaddr, 21235767Sgibbs bus_addr_t highaddr, bus_dma_filter_t *filter, 21335767Sgibbs void *filterarg, bus_size_t maxsize, int nsegments, 214117126Sscottl bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 215117126Sscottl void *lockfuncarg, bus_dma_tag_t *dmat) 21632516Sgibbs{ 21732516Sgibbs bus_dma_tag_t newtag; 21832516Sgibbs int error = 0; 21932516Sgibbs 220131529Sscottl /* Basic sanity checking */ 221131529Sscottl if (boundary != 0 && boundary < maxsegsz) 222131529Sscottl maxsegsz = boundary; 223131529Sscottl 22432516Sgibbs /* Return a NULL tag on failure */ 22532516Sgibbs *dmat = NULL; 22632516Sgibbs 227137460Sscottl newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 228137460Sscottl M_ZERO | M_NOWAIT); 229136805Srwatson if (newtag == NULL) { 230136805Srwatson CTR3(KTR_BUSDMA, "bus_dma_tag_create returned tag %p tag " 231136805Srwatson "flags 0x%x error %d", newtag, 0, error); 23232516Sgibbs return (ENOMEM); 233136805Srwatson } 23432516Sgibbs 23532516Sgibbs newtag->parent = parent; 23648449Smjacob newtag->alignment = alignment; 23732516Sgibbs newtag->boundary = boundary; 238112569Sjake newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 239112569Sjake newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 240112569Sjake (PAGE_SIZE - 1); 24132516Sgibbs newtag->filter = filter; 24232516Sgibbs newtag->filterarg = filterarg; 24332516Sgibbs newtag->maxsize = maxsize; 24432516Sgibbs newtag->nsegments = nsegments; 24532516Sgibbs newtag->maxsegsz = maxsegsz; 24632516Sgibbs newtag->flags = flags; 24732516Sgibbs newtag->ref_count = 1; /* Count ourself */ 24832516Sgibbs newtag->map_count = 0; 249117126Sscottl if (lockfunc != NULL) { 250117126Sscottl newtag->lockfunc = lockfunc; 251117126Sscottl newtag->lockfuncarg = lockfuncarg; 252117126Sscottl } else { 253117126Sscottl newtag->lockfunc = dflt_lock; 254117126Sscottl newtag->lockfuncarg = NULL; 255117126Sscottl } 256118246Sscottl newtag->segments = NULL; 257118246Sscottl 25832516Sgibbs /* Take into account any restrictions imposed by our parent tag */ 25932516Sgibbs if (parent != NULL) { 26032516Sgibbs newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 26132516Sgibbs newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 262134934Sscottl if (newtag->boundary == 0) 263134934Sscottl newtag->boundary = parent->boundary; 264134934Sscottl else if (parent->boundary != 0) 265134934Sscottl newtag->boundary = MIN(parent->boundary, 266134934Sscottl newtag->boundary); 26732516Sgibbs if (newtag->filter == NULL) { 26832516Sgibbs /* 26932516Sgibbs * Short circuit looking at our parent directly 27035256Sdes * since we have encapsulated all of its information 27132516Sgibbs */ 27232516Sgibbs newtag->filter = parent->filter; 27332516Sgibbs newtag->filterarg = parent->filterarg; 27432516Sgibbs newtag->parent = parent->parent; 27532516Sgibbs } 276112436Smux if (newtag->parent != NULL) 277112436Smux atomic_add_int(&parent->ref_count, 1); 27832516Sgibbs } 279137965Sscottl 280137965Sscottl if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) 281137965Sscottl || newtag->alignment > 1 || newtag->boundary > 0) 282137965Sscottl newtag->flags |= BUS_DMA_COULD_BOUNCE; 283137965Sscottl 284137965Sscottl if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 285112569Sjake (flags & BUS_DMA_ALLOCNOW) != 0) { 286137965Sscottl struct bounce_zone *bz; 287137965Sscottl 28832516Sgibbs /* Must bounce */ 28932516Sgibbs 290137965Sscottl if ((error = alloc_bounce_zone(newtag)) != 0) 291137965Sscottl return (error); 292137965Sscottl bz = newtag->bounce_zone; 293137965Sscottl 29432516Sgibbs if (lowaddr > bounce_lowaddr) { 29532516Sgibbs /* 29632516Sgibbs * Go through the pool and kill any pages 29732516Sgibbs * that don't reside below lowaddr. 29832516Sgibbs */ 29935767Sgibbs panic("bus_dma_tag_create: page reallocation " 30032516Sgibbs "not implemented"); 30132516Sgibbs } 302137965Sscottl if (ptoa(bz->total_bpages) < maxsize) { 30332516Sgibbs int pages; 30432516Sgibbs 305137965Sscottl pages = atop(maxsize) - bz->total_bpages; 30632516Sgibbs 30732516Sgibbs /* Add pages to our bounce pool */ 30832516Sgibbs if (alloc_bounce_pages(newtag, pages) < pages) 30932516Sgibbs error = ENOMEM; 31032516Sgibbs } 31135767Sgibbs /* Performed initial allocation */ 31235767Sgibbs newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 31332516Sgibbs } 31432516Sgibbs 31532516Sgibbs if (error != 0) { 31632516Sgibbs free(newtag, M_DEVBUF); 31732516Sgibbs } else { 31832516Sgibbs *dmat = newtag; 31932516Sgibbs } 320136805Srwatson CTR3(KTR_BUSDMA, "bus_dma_tag_create returned tag %p tag flags 0x%x " 321136805Srwatson "error %d", newtag, (newtag != NULL ? newtag->flags : 0), error); 32232516Sgibbs return (error); 32332516Sgibbs} 32432516Sgibbs 32532516Sgibbsint 32632516Sgibbsbus_dma_tag_destroy(bus_dma_tag_t dmat) 32732516Sgibbs{ 328136805Srwatson bus_dma_tag_t dmat_copy; 329136805Srwatson int error; 330136805Srwatson 331136805Srwatson error = 0; 332136805Srwatson dmat_copy = dmat; 333136805Srwatson 33432516Sgibbs if (dmat != NULL) { 33532516Sgibbs 336136805Srwatson if (dmat->map_count != 0) { 337136805Srwatson error = EBUSY; 338136805Srwatson goto out; 339136805Srwatson } 34032516Sgibbs 34132516Sgibbs while (dmat != NULL) { 34232516Sgibbs bus_dma_tag_t parent; 34332516Sgibbs 34432516Sgibbs parent = dmat->parent; 345112436Smux atomic_subtract_int(&dmat->ref_count, 1); 34632516Sgibbs if (dmat->ref_count == 0) { 347118246Sscottl if (dmat->segments != NULL) 348118246Sscottl free(dmat->segments, M_DEVBUF); 34932516Sgibbs free(dmat, M_DEVBUF); 35040029Sgibbs /* 35140029Sgibbs * Last reference count, so 35240029Sgibbs * release our reference 35340029Sgibbs * count on our parent. 35440029Sgibbs */ 35540029Sgibbs dmat = parent; 35640029Sgibbs } else 35740029Sgibbs dmat = NULL; 35832516Sgibbs } 35932516Sgibbs } 360136805Srwatsonout: 361136805Srwatson CTR2(KTR_BUSDMA, "bus_dma_tag_destroy tag %p error %d", dmat_copy, 362136805Srwatson error); 363136805Srwatson return (error); 36432516Sgibbs} 36532516Sgibbs 36632516Sgibbs/* 36732516Sgibbs * Allocate a handle for mapping from kva/uva/physical 36832516Sgibbs * address space into bus device space. 36932516Sgibbs */ 37032516Sgibbsint 37132516Sgibbsbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 37232516Sgibbs{ 37332516Sgibbs int error; 37432516Sgibbs 37532516Sgibbs error = 0; 37632516Sgibbs 377118246Sscottl if (dmat->segments == NULL) { 378118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 379118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 380118246Sscottl M_NOWAIT); 381136805Srwatson if (dmat->segments == NULL) { 382136805Srwatson CTR2(KTR_BUSDMA, "bus_dmamap_create: tag %p error %d", 383136805Srwatson dmat, ENOMEM); 384118246Sscottl return (ENOMEM); 385136805Srwatson } 386118246Sscottl } 387118246Sscottl 388131529Sscottl /* 389131529Sscottl * Bouncing might be required if the driver asks for an active 390131529Sscottl * exclusion region, a data alignment that is stricter than 1, and/or 391131529Sscottl * an active address boundary. 392131529Sscottl */ 393137965Sscottl if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 394137445Sscottl 39532516Sgibbs /* Must bounce */ 39632516Sgibbs int maxpages; 39732516Sgibbs 398137965Sscottl if (dmat->bounce_zone == NULL) { 399137965Sscottl if ((error = alloc_bounce_zone(dmat)) != 0) 400137965Sscottl return (error); 401137965Sscottl } 402137965Sscottl 40332516Sgibbs *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 40469781Sdwmalone M_NOWAIT | M_ZERO); 405136805Srwatson if (*mapp == NULL) { 406136805Srwatson CTR2(KTR_BUSDMA, "bus_dmamap_create: tag %p error %d", 407136805Srwatson dmat, ENOMEM); 40835767Sgibbs return (ENOMEM); 409136805Srwatson } 41069781Sdwmalone 41169781Sdwmalone /* Initialize the new map */ 41269781Sdwmalone STAILQ_INIT(&((*mapp)->bpages)); 41369781Sdwmalone 41432516Sgibbs /* 41532516Sgibbs * Attempt to add pages to our pool on a per-instance 41632516Sgibbs * basis up to a sane limit. 41732516Sgibbs */ 41832516Sgibbs maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->lowaddr)); 41935767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 420131529Sscottl || (dmat->map_count > 0 && total_bpages < maxpages)) { 42132516Sgibbs int pages; 42232516Sgibbs 42335767Sgibbs if (dmat->lowaddr > bounce_lowaddr) { 42435767Sgibbs /* 42535767Sgibbs * Go through the pool and kill any pages 42635767Sgibbs * that don't reside below lowaddr. 42735767Sgibbs */ 42835767Sgibbs panic("bus_dmamap_create: page reallocation " 42935767Sgibbs "not implemented"); 43035767Sgibbs } 431113228Sjake pages = MAX(atop(dmat->maxsize), 1); 43232516Sgibbs pages = MIN(maxpages - total_bpages, pages); 433113228Sjake if (alloc_bounce_pages(dmat, pages) < pages) 434113228Sjake error = ENOMEM; 43535767Sgibbs 43635767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 43735767Sgibbs if (error == 0) 43835767Sgibbs dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 43935767Sgibbs } else { 44035767Sgibbs error = 0; 44135767Sgibbs } 44232516Sgibbs } 44332516Sgibbs } else { 44440029Sgibbs *mapp = NULL; 44532516Sgibbs } 44632516Sgibbs if (error == 0) 44732516Sgibbs dmat->map_count++; 448136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamap_create: tag %p tag flags 0x%x error %d", 449136805Srwatson dmat, dmat->flags, error); 45032516Sgibbs return (error); 45132516Sgibbs} 45232516Sgibbs 45332516Sgibbs/* 45432516Sgibbs * Destroy a handle for mapping from kva/uva/physical 45532516Sgibbs * address space into bus device space. 45632516Sgibbs */ 45732516Sgibbsint 45832516Sgibbsbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 45932516Sgibbs{ 460117136Smux if (map != NULL && map != &nobounce_dmamap) { 461136805Srwatson if (STAILQ_FIRST(&map->bpages) != NULL) { 462136805Srwatson CTR2(KTR_BUSDMA, "bus_dmamap_destroy: tag %p error %d", 463136805Srwatson dmat, EBUSY); 46432516Sgibbs return (EBUSY); 465136805Srwatson } 46632516Sgibbs free(map, M_DEVBUF); 46732516Sgibbs } 46832516Sgibbs dmat->map_count--; 469136805Srwatson CTR1(KTR_BUSDMA, "bus_dmamap_destroy: tag %p error 0", dmat); 47032516Sgibbs return (0); 47132516Sgibbs} 47232516Sgibbs 47335767Sgibbs 47435767Sgibbs/* 47535767Sgibbs * Allocate a piece of memory that can be efficiently mapped into 47635767Sgibbs * bus device space based on the constraints lited in the dma tag. 47735767Sgibbs * A dmamap to for use with dmamap_load is also allocated. 47835767Sgibbs */ 47935767Sgibbsint 480115316Sscottlbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 481115316Sscottl bus_dmamap_t *mapp) 48235767Sgibbs{ 483118081Smux int mflags; 484118081Smux 485118081Smux if (flags & BUS_DMA_NOWAIT) 486118081Smux mflags = M_NOWAIT; 487118081Smux else 488118081Smux mflags = M_WAITOK; 489118081Smux if (flags & BUS_DMA_ZERO) 490118081Smux mflags |= M_ZERO; 491118081Smux 49235767Sgibbs /* If we succeed, no mapping/bouncing will be required */ 49340029Sgibbs *mapp = NULL; 49435767Sgibbs 495118246Sscottl if (dmat->segments == NULL) { 496118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 497118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 498118246Sscottl M_NOWAIT); 499136805Srwatson if (dmat->segments == NULL) { 500136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamem_alloc: tag %p tag " 501136805Srwatson "flags 0x%x error %d", dmat, dmat->flags, ENOMEM); 502118246Sscottl return (ENOMEM); 503136805Srwatson } 504118246Sscottl } 505118246Sscottl 506115316Sscottl if ((dmat->maxsize <= PAGE_SIZE) && 507112569Sjake dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 508118081Smux *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 50935767Sgibbs } else { 51035767Sgibbs /* 51135767Sgibbs * XXX Use Contigmalloc until it is merged into this facility 51235767Sgibbs * and handles multi-seg allocations. Nobody is doing 51335767Sgibbs * multi-seg allocations yet though. 514131529Sscottl * XXX Certain AGP hardware does. 51535767Sgibbs */ 516118081Smux *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 51748449Smjacob 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 51848449Smjacob dmat->boundary); 51935767Sgibbs } 520136805Srwatson if (*vaddr == NULL) { 521136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamem_alloc: tag %p tag flags 0x%x " 522136805Srwatson "error %d", dmat, dmat->flags, ENOMEM); 52335767Sgibbs return (ENOMEM); 524136805Srwatson } 525136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamem_alloc: tag %p tag flags 0x%x error %d", 526136805Srwatson dmat, dmat->flags, ENOMEM); 52735767Sgibbs return (0); 52835767Sgibbs} 52935767Sgibbs 53035767Sgibbs/* 53135767Sgibbs * Free a piece of memory and it's allociated dmamap, that was allocated 53295076Salfred * via bus_dmamem_alloc. Make the same choice for free/contigfree. 53335767Sgibbs */ 53435767Sgibbsvoid 535115316Sscottlbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 53635767Sgibbs{ 53735767Sgibbs /* 53835767Sgibbs * dmamem does not need to be bounced, so the map should be 53935767Sgibbs * NULL 54035767Sgibbs */ 54149859Sgibbs if (map != NULL) 54235767Sgibbs panic("bus_dmamem_free: Invalid map freed\n"); 543115316Sscottl if ((dmat->maxsize <= PAGE_SIZE) 544115316Sscottl && dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 54540029Sgibbs free(vaddr, M_DEVBUF); 546112196Smux else { 547115316Sscottl contigfree(vaddr, dmat->maxsize, M_DEVBUF); 548112196Smux } 549136805Srwatson CTR2(KTR_BUSDMA, "bus_dmamem_free: tag %p flags 0x%x", dmat, 550136805Srwatson dmat->flags); 55135767Sgibbs} 55235767Sgibbs 55332516Sgibbs/* 554104486Ssam * Utility function to load a linear buffer. lastaddrp holds state 555104486Ssam * between invocations (for multiple-buffer loads). segp contains 556104486Ssam * the starting segment on entrace, and the ending segment on exit. 557104486Ssam * first indicates if this is the first invocation of this function. 558104486Ssam */ 559137142Sscottlstatic __inline int 560104486Ssam_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 561113228Sjake bus_dmamap_t map, 562104486Ssam void *buf, bus_size_t buflen, 563137142Sscottl pmap_t pmap, 564104486Ssam int flags, 565113228Sjake bus_addr_t *lastaddrp, 566104486Ssam int *segp, 567104486Ssam int first) 568104486Ssam{ 569118246Sscottl bus_dma_segment_t *segs; 570104486Ssam bus_size_t sgsize; 571104486Ssam bus_addr_t curaddr, lastaddr, baddr, bmask; 572113228Sjake vm_offset_t vaddr; 573113228Sjake bus_addr_t paddr; 574113228Sjake int needbounce = 0; 575104486Ssam int seg; 576104486Ssam 577118246Sscottl segs = dmat->segments; 578118246Sscottl 579113228Sjake if (map == NULL) 580113228Sjake map = &nobounce_dmamap; 581113228Sjake 582137142Sscottl if ((map != &nobounce_dmamap && map->pagesneeded == 0) 583137965Sscottl && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { 584113228Sjake vm_offset_t vendaddr; 585113228Sjake 586137142Sscottl CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 587137142Sscottl "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), 588137142Sscottl dmat->boundary, dmat->alignment); 589137142Sscottl CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", 590137142Sscottl map, &nobounce_dmamap, map->pagesneeded); 591113228Sjake /* 592113228Sjake * Count the number of bounce pages 593113228Sjake * needed in order to complete this transfer 594113228Sjake */ 595113228Sjake vaddr = trunc_page((vm_offset_t)buf); 596113228Sjake vendaddr = (vm_offset_t)buf + buflen; 597113228Sjake 598113228Sjake while (vaddr < vendaddr) { 599113228Sjake paddr = pmap_kextract(vaddr); 600137894Sscottl if (run_filter(dmat, paddr) != 0) { 601113228Sjake needbounce = 1; 602113228Sjake map->pagesneeded++; 603113228Sjake } 604113228Sjake vaddr += PAGE_SIZE; 605113228Sjake } 606137142Sscottl CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 607113228Sjake } 608113228Sjake 609113228Sjake /* Reserve Necessary Bounce Pages */ 610113228Sjake if (map->pagesneeded != 0) { 611113228Sjake mtx_lock(&bounce_lock); 612113472Ssimokawa if (flags & BUS_DMA_NOWAIT) { 613113472Ssimokawa if (reserve_bounce_pages(dmat, map, 0) != 0) { 614113472Ssimokawa mtx_unlock(&bounce_lock); 615113472Ssimokawa return (ENOMEM); 616113472Ssimokawa } 617113472Ssimokawa } else { 618113472Ssimokawa if (reserve_bounce_pages(dmat, map, 1) != 0) { 619132545Sscottl /* Queue us for resources */ 620113472Ssimokawa map->dmat = dmat; 621113472Ssimokawa map->buf = buf; 622113472Ssimokawa map->buflen = buflen; 623113472Ssimokawa STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 624117136Smux map, links); 625113472Ssimokawa mtx_unlock(&bounce_lock); 626113472Ssimokawa return (EINPROGRESS); 627113472Ssimokawa } 628113228Sjake } 629113228Sjake mtx_unlock(&bounce_lock); 630113228Sjake } 631113228Sjake 632137142Sscottl vaddr = (vm_offset_t)buf; 633104486Ssam lastaddr = *lastaddrp; 634113228Sjake bmask = ~(dmat->boundary - 1); 635104486Ssam 636104486Ssam for (seg = *segp; buflen > 0 ; ) { 637104486Ssam /* 638104486Ssam * Get the physical address for this segment. 639104486Ssam */ 640104486Ssam if (pmap) 641104486Ssam curaddr = pmap_extract(pmap, vaddr); 642104486Ssam else 643104486Ssam curaddr = pmap_kextract(vaddr); 644104486Ssam 645104486Ssam /* 646104486Ssam * Compute the segment size, and adjust counts. 647104486Ssam */ 648104486Ssam sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 649104486Ssam if (buflen < sgsize) 650104486Ssam sgsize = buflen; 651104486Ssam 652104486Ssam /* 653104486Ssam * Make sure we don't cross any boundaries. 654104486Ssam */ 655104486Ssam if (dmat->boundary > 0) { 656104486Ssam baddr = (curaddr + dmat->boundary) & bmask; 657104486Ssam if (sgsize > (baddr - curaddr)) 658104486Ssam sgsize = (baddr - curaddr); 659104486Ssam } 660104486Ssam 661137894Sscottl if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) 662113228Sjake curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 663113228Sjake 664104486Ssam /* 665104486Ssam * Insert chunk into a segment, coalescing with 666104486Ssam * previous segment if possible. 667104486Ssam */ 668104486Ssam if (first) { 669104486Ssam segs[seg].ds_addr = curaddr; 670104486Ssam segs[seg].ds_len = sgsize; 671104486Ssam first = 0; 672104486Ssam } else { 673113228Sjake if (needbounce == 0 && curaddr == lastaddr && 674104486Ssam (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 675104486Ssam (dmat->boundary == 0 || 676104486Ssam (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 677104486Ssam segs[seg].ds_len += sgsize; 678104486Ssam else { 679104486Ssam if (++seg >= dmat->nsegments) 680104486Ssam break; 681104486Ssam segs[seg].ds_addr = curaddr; 682104486Ssam segs[seg].ds_len = sgsize; 683104486Ssam } 684104486Ssam } 685104486Ssam 686104486Ssam lastaddr = curaddr + sgsize; 687104486Ssam vaddr += sgsize; 688104486Ssam buflen -= sgsize; 689104486Ssam } 690104486Ssam 691104486Ssam *segp = seg; 692104486Ssam *lastaddrp = lastaddr; 693104486Ssam 694104486Ssam /* 695104486Ssam * Did we fit? 696104486Ssam */ 697104486Ssam return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 698104486Ssam} 699104486Ssam 700104486Ssam/* 701113459Ssimokawa * Map the buffer buf into bus space using the dmamap map. 702113459Ssimokawa */ 703113459Ssimokawaint 704113459Ssimokawabus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 705113459Ssimokawa bus_size_t buflen, bus_dmamap_callback_t *callback, 706113459Ssimokawa void *callback_arg, int flags) 707113459Ssimokawa{ 708113492Smux bus_addr_t lastaddr = 0; 709113459Ssimokawa int error, nsegs = 0; 710113459Ssimokawa 711113472Ssimokawa if (map != NULL) { 712113472Ssimokawa flags |= BUS_DMA_WAITOK; 713113472Ssimokawa map->callback = callback; 714113472Ssimokawa map->callback_arg = callback_arg; 715113472Ssimokawa } 716113472Ssimokawa 717118246Sscottl error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 718118246Sscottl &lastaddr, &nsegs, 1); 719113459Ssimokawa 720136805Srwatson if (error == EINPROGRESS) { 721136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x " 722136805Srwatson "error %d", dmat, dmat->flags, error); 723113492Smux return (error); 724136805Srwatson } 725113472Ssimokawa 726113459Ssimokawa if (error) 727118246Sscottl (*callback)(callback_arg, dmat->segments, 0, error); 728113459Ssimokawa else 729118246Sscottl (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 730113459Ssimokawa 731136805Srwatson CTR2(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x error 0", 732136805Srwatson dmat, dmat->flags); 733113459Ssimokawa return (0); 734113459Ssimokawa} 735113459Ssimokawa 736113459Ssimokawa 737113459Ssimokawa/* 738104486Ssam * Like _bus_dmamap_load(), but for mbufs. 739104486Ssam */ 740104486Ssamint 741104486Ssambus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 742104486Ssam struct mbuf *m0, 743104486Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 744104486Ssam int flags) 745104486Ssam{ 746104486Ssam int nsegs, error; 747104486Ssam 748117136Smux M_ASSERTPKTHDR(m0); 749104486Ssam 750113472Ssimokawa flags |= BUS_DMA_NOWAIT; 751104486Ssam nsegs = 0; 752104486Ssam error = 0; 753104486Ssam if (m0->m_pkthdr.len <= dmat->maxsize) { 754104486Ssam int first = 1; 755113228Sjake bus_addr_t lastaddr = 0; 756104486Ssam struct mbuf *m; 757104486Ssam 758104486Ssam for (m = m0; m != NULL && error == 0; m = m->m_next) { 759110335Sharti if (m->m_len > 0) { 760113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 761110335Sharti m->m_data, m->m_len, 762110335Sharti NULL, flags, &lastaddr, 763110335Sharti &nsegs, first); 764110335Sharti first = 0; 765110335Sharti } 766104486Ssam } 767104486Ssam } else { 768104486Ssam error = EINVAL; 769104486Ssam } 770104486Ssam 771104486Ssam if (error) { 772104486Ssam /* force "no valid mappings" in callback */ 773118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 774104486Ssam } else { 775118246Sscottl (*callback)(callback_arg, dmat->segments, 776104486Ssam nsegs+1, m0->m_pkthdr.len, error); 777104486Ssam } 778136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x " 779136805Srwatson "error %d", dmat, dmat->flags, error); 780104486Ssam return (error); 781104486Ssam} 782104486Ssam 783104486Ssam/* 784104486Ssam * Like _bus_dmamap_load(), but for uios. 785104486Ssam */ 786104486Ssamint 787104486Ssambus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 788104486Ssam struct uio *uio, 789104486Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 790104486Ssam int flags) 791104486Ssam{ 792113228Sjake bus_addr_t lastaddr; 793104486Ssam int nsegs, error, first, i; 794104486Ssam bus_size_t resid; 795104486Ssam struct iovec *iov; 796137142Sscottl pmap_t pmap; 797104486Ssam 798113472Ssimokawa flags |= BUS_DMA_NOWAIT; 799104486Ssam resid = uio->uio_resid; 800104486Ssam iov = uio->uio_iov; 801104486Ssam 802104486Ssam if (uio->uio_segflg == UIO_USERSPACE) { 803137142Sscottl KASSERT(uio->uio_td != NULL, 804104486Ssam ("bus_dmamap_load_uio: USERSPACE but no proc")); 805137142Sscottl pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 806137142Sscottl } else 807137142Sscottl pmap = NULL; 808104486Ssam 809104486Ssam nsegs = 0; 810104486Ssam error = 0; 811104486Ssam first = 1; 812104486Ssam for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 813104486Ssam /* 814104486Ssam * Now at the first iovec to load. Load each iovec 815104486Ssam * until we have exhausted the residual count. 816104486Ssam */ 817104486Ssam bus_size_t minlen = 818104486Ssam resid < iov[i].iov_len ? resid : iov[i].iov_len; 819104486Ssam caddr_t addr = (caddr_t) iov[i].iov_base; 820104486Ssam 821110335Sharti if (minlen > 0) { 822113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 823110335Sharti addr, minlen, 824137142Sscottl pmap, flags, &lastaddr, &nsegs, first); 825110335Sharti first = 0; 826104486Ssam 827110335Sharti resid -= minlen; 828110335Sharti } 829104486Ssam } 830104486Ssam 831104486Ssam if (error) { 832104486Ssam /* force "no valid mappings" in callback */ 833118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 834104486Ssam } else { 835118246Sscottl (*callback)(callback_arg, dmat->segments, 836104486Ssam nsegs+1, uio->uio_resid, error); 837104486Ssam } 838136805Srwatson CTR3(KTR_BUSDMA, "bus_dmamap_load_uio: tag %p tag flags 0x%x " 839136805Srwatson "error %d", dmat, dmat->flags, error); 840104486Ssam return (error); 841104486Ssam} 842104486Ssam 843104486Ssam/* 84432516Sgibbs * Release the mapping held by map. 84532516Sgibbs */ 84632516Sgibbsvoid 84732516Sgibbs_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 84832516Sgibbs{ 84932516Sgibbs struct bounce_page *bpage; 85032516Sgibbs 85132516Sgibbs while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 85232516Sgibbs STAILQ_REMOVE_HEAD(&map->bpages, links); 85332516Sgibbs free_bounce_page(dmat, bpage); 85432516Sgibbs } 85532516Sgibbs} 85632516Sgibbs 85732516Sgibbsvoid 858115343Sscottl_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 85932516Sgibbs{ 86032516Sgibbs struct bounce_page *bpage; 86132516Sgibbs 86232516Sgibbs if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 86332516Sgibbs /* 86432516Sgibbs * Handle data bouncing. We might also 86532516Sgibbs * want to add support for invalidating 86632516Sgibbs * the caches on broken hardware 86732516Sgibbs */ 868137445Sscottl dmat->bounce_zone->total_bounced++; 869136805Srwatson CTR3(KTR_BUSDMA, "_bus_dmamap_sync: tag %p tag flags 0x%x " 870136805Srwatson "op 0x%x performing bounce", op, dmat, dmat->flags); 871131529Sscottl 872113347Smux if (op & BUS_DMASYNC_PREWRITE) { 87332516Sgibbs while (bpage != NULL) { 87432516Sgibbs bcopy((void *)bpage->datavaddr, 87532516Sgibbs (void *)bpage->vaddr, 87632516Sgibbs bpage->datacount); 87732516Sgibbs bpage = STAILQ_NEXT(bpage, links); 87832516Sgibbs } 879113347Smux } 88032516Sgibbs 881113347Smux if (op & BUS_DMASYNC_POSTREAD) { 88232516Sgibbs while (bpage != NULL) { 88332516Sgibbs bcopy((void *)bpage->vaddr, 88432516Sgibbs (void *)bpage->datavaddr, 88532516Sgibbs bpage->datacount); 88632516Sgibbs bpage = STAILQ_NEXT(bpage, links); 88732516Sgibbs } 88832516Sgibbs } 88932516Sgibbs } 89032516Sgibbs} 89132516Sgibbs 892112346Smuxstatic void 893112346Smuxinit_bounce_pages(void *dummy __unused) 894112346Smux{ 895112346Smux 896112346Smux total_bpages = 0; 897137445Sscottl STAILQ_INIT(&bounce_zone_list); 898112346Smux STAILQ_INIT(&bounce_map_waitinglist); 899112346Smux STAILQ_INIT(&bounce_map_callbacklist); 900112346Smux mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 901112346Smux} 902112346SmuxSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 903112346Smux 904137445Sscottlstatic struct sysctl_ctx_list * 905137445Sscottlbusdma_sysctl_tree(struct bounce_zone *bz) 906137445Sscottl{ 907137445Sscottl return (&bz->sysctl_tree); 908137445Sscottl} 909137445Sscottl 910137445Sscottlstatic struct sysctl_oid * 911137445Sscottlbusdma_sysctl_tree_top(struct bounce_zone *bz) 912137445Sscottl{ 913137445Sscottl return (bz->sysctl_tree_top); 914137445Sscottl} 915137445Sscottl 916137965Sscottlstatic int 917137445Sscottlalloc_bounce_zone(bus_dma_tag_t dmat) 918137445Sscottl{ 919137445Sscottl struct bounce_zone *bz; 920137445Sscottl 921137965Sscottl /* Check to see if we already have a suitable zone */ 922137965Sscottl STAILQ_FOREACH(bz, &bounce_zone_list, links) { 923137965Sscottl if ((dmat->alignment <= bz->alignment) 924137965Sscottl && (dmat->boundary <= bz->boundary) 925137965Sscottl && (dmat->lowaddr >= bz->lowaddr)) { 926137965Sscottl dmat->bounce_zone = bz; 927137965Sscottl return (0); 928137965Sscottl } 929137965Sscottl } 930137965Sscottl 931137445Sscottl if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 932137445Sscottl M_NOWAIT | M_ZERO)) == NULL) 933137965Sscottl return (ENOMEM); 934137445Sscottl 935137445Sscottl STAILQ_INIT(&bz->bounce_page_list); 936137445Sscottl bz->free_bpages = 0; 937137445Sscottl bz->reserved_bpages = 0; 938137445Sscottl bz->active_bpages = 0; 939137445Sscottl bz->lowaddr = dmat->lowaddr; 940137445Sscottl bz->alignment = dmat->alignment; 941137445Sscottl bz->boundary = dmat->boundary; 942137445Sscottl snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 943137445Sscottl busdma_zonecount++; 944137460Sscottl snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 945137445Sscottl STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 946137965Sscottl dmat->bounce_zone = bz; 947137445Sscottl 948137445Sscottl sysctl_ctx_init(&bz->sysctl_tree); 949137445Sscottl bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 950137445Sscottl SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 951137445Sscottl CTLFLAG_RD, 0, ""); 952137445Sscottl if (bz->sysctl_tree_top == NULL) { 953137445Sscottl sysctl_ctx_free(&bz->sysctl_tree); 954137965Sscottl return (0); /* XXX error code? */ 955137445Sscottl } 956137445Sscottl 957137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 958137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 959137965Sscottl "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 960137965Sscottl "Totoal bounce pages"); 961137965Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 962137965Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 963137445Sscottl "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 964137445Sscottl "Free bounce pages"); 965137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 966137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 967137445Sscottl "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 968137445Sscottl "Reserved bounce pages"); 969137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 970137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 971137445Sscottl "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 972137445Sscottl "Active bounce pages"); 973137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 974137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 975137445Sscottl "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 976137445Sscottl "Total bounce requests"); 977137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 978137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 979137445Sscottl "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 980137445Sscottl "Total bounce requests that were deferred"); 981137445Sscottl SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 982137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 983137445Sscottl "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 984137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 985137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 986137445Sscottl "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 987137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 988137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 989137445Sscottl "boundary", CTLFLAG_RD, &bz->boundary, 0, ""); 990137445Sscottl 991137965Sscottl return (0); 992137445Sscottl} 993137445Sscottl 99432516Sgibbsstatic int 99532516Sgibbsalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 99632516Sgibbs{ 997137445Sscottl struct bounce_zone *bz; 99832516Sgibbs int count; 99932516Sgibbs 1000137445Sscottl bz = dmat->bounce_zone; 100132516Sgibbs count = 0; 100232516Sgibbs while (numpages > 0) { 100332516Sgibbs struct bounce_page *bpage; 100432516Sgibbs 100532516Sgibbs bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 100669781Sdwmalone M_NOWAIT | M_ZERO); 100732516Sgibbs 100832516Sgibbs if (bpage == NULL) 100932516Sgibbs break; 101032516Sgibbs bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 101132516Sgibbs M_NOWAIT, 0ul, 1012137445Sscottl bz->lowaddr, 1013132545Sscottl PAGE_SIZE, 1014137445Sscottl bz->boundary); 1015102241Sarchie if (bpage->vaddr == 0) { 101632516Sgibbs free(bpage, M_DEVBUF); 101732516Sgibbs break; 101832516Sgibbs } 101932516Sgibbs bpage->busaddr = pmap_kextract(bpage->vaddr); 1020112346Smux mtx_lock(&bounce_lock); 1021137445Sscottl STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 102232516Sgibbs total_bpages++; 1023137965Sscottl bz->total_bpages++; 1024137445Sscottl bz->free_bpages++; 1025112346Smux mtx_unlock(&bounce_lock); 102632516Sgibbs count++; 102732516Sgibbs numpages--; 102832516Sgibbs } 102932516Sgibbs return (count); 103032516Sgibbs} 103132516Sgibbs 103232516Sgibbsstatic int 1033113228Sjakereserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 103432516Sgibbs{ 1035137445Sscottl struct bounce_zone *bz; 103632516Sgibbs int pages; 103732516Sgibbs 1038112346Smux mtx_assert(&bounce_lock, MA_OWNED); 1039137445Sscottl bz = dmat->bounce_zone; 1040137445Sscottl pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1041113228Sjake if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1042113228Sjake return (map->pagesneeded - (map->pagesreserved + pages)); 1043137445Sscottl bz->free_bpages -= pages; 1044137445Sscottl bz->reserved_bpages += pages; 104532516Sgibbs map->pagesreserved += pages; 104632516Sgibbs pages = map->pagesneeded - map->pagesreserved; 104732516Sgibbs 104832516Sgibbs return (pages); 104932516Sgibbs} 105032516Sgibbs 1051112569Sjakestatic bus_addr_t 105232516Sgibbsadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 105332516Sgibbs bus_size_t size) 105432516Sgibbs{ 1055137445Sscottl struct bounce_zone *bz; 105632516Sgibbs struct bounce_page *bpage; 105732516Sgibbs 1058137445Sscottl KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1059113228Sjake KASSERT(map != NULL && map != &nobounce_dmamap, 1060113228Sjake ("add_bounce_page: bad map %p", map)); 1061113228Sjake 1062137445Sscottl bz = dmat->bounce_zone; 106332516Sgibbs if (map->pagesneeded == 0) 106432516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 106532516Sgibbs map->pagesneeded--; 106632516Sgibbs 106732516Sgibbs if (map->pagesreserved == 0) 106832516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 106932516Sgibbs map->pagesreserved--; 107032516Sgibbs 1071112346Smux mtx_lock(&bounce_lock); 1072137445Sscottl bpage = STAILQ_FIRST(&bz->bounce_page_list); 107332516Sgibbs if (bpage == NULL) 107432516Sgibbs panic("add_bounce_page: free page list is empty"); 107532516Sgibbs 1076137445Sscottl STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1077137445Sscottl bz->reserved_bpages--; 1078137445Sscottl bz->active_bpages++; 1079112346Smux mtx_unlock(&bounce_lock); 108032516Sgibbs 108132516Sgibbs bpage->datavaddr = vaddr; 108232516Sgibbs bpage->datacount = size; 108332516Sgibbs STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 108432516Sgibbs return (bpage->busaddr); 108532516Sgibbs} 108632516Sgibbs 108732516Sgibbsstatic void 108832516Sgibbsfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 108932516Sgibbs{ 109032516Sgibbs struct bus_dmamap *map; 1091137445Sscottl struct bounce_zone *bz; 109232516Sgibbs 1093137445Sscottl bz = dmat->bounce_zone; 109432516Sgibbs bpage->datavaddr = 0; 109532516Sgibbs bpage->datacount = 0; 109632516Sgibbs 1097112346Smux mtx_lock(&bounce_lock); 1098137445Sscottl STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1099137445Sscottl bz->free_bpages++; 1100137445Sscottl bz->active_bpages--; 110132516Sgibbs if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1102113228Sjake if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 110332516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 110432516Sgibbs STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 110532516Sgibbs map, links); 110632516Sgibbs busdma_swi_pending = 1; 1107137445Sscottl bz->total_deferred++; 110888900Sjhb swi_sched(vm_ih, 0); 110932516Sgibbs } 111032516Sgibbs } 1111112346Smux mtx_unlock(&bounce_lock); 111232516Sgibbs} 111332516Sgibbs 111432516Sgibbsvoid 111595076Salfredbusdma_swi(void) 111632516Sgibbs{ 1117117126Sscottl bus_dma_tag_t dmat; 111832516Sgibbs struct bus_dmamap *map; 111932516Sgibbs 1120112346Smux mtx_lock(&bounce_lock); 112132516Sgibbs while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 112232516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1123112346Smux mtx_unlock(&bounce_lock); 1124117136Smux dmat = map->dmat; 1125117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 112632516Sgibbs bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 112732516Sgibbs map->callback, map->callback_arg, /*flags*/0); 1128117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1129112346Smux mtx_lock(&bounce_lock); 113032516Sgibbs } 1131112346Smux mtx_unlock(&bounce_lock); 113232516Sgibbs} 1133