busdma_machdep.c revision 169799
1139724Simp/*- 240029Sgibbs * Copyright (c) 1997, 1998 Justin T. Gibbs. 332516Sgibbs * All rights reserved. 432516Sgibbs * 532516Sgibbs * Redistribution and use in source and binary forms, with or without 632516Sgibbs * modification, are permitted provided that the following conditions 732516Sgibbs * are met: 832516Sgibbs * 1. Redistributions of source code must retain the above copyright 932516Sgibbs * notice, this list of conditions, and the following disclaimer, 1032516Sgibbs * without modification, immediately at the beginning of the file. 1132516Sgibbs * 2. The name of the author may not be used to endorse or promote products 1232516Sgibbs * derived from this software without specific prior written permission. 1332516Sgibbs * 1432516Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1532516Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1632516Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1732516Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 1832516Sgibbs * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1932516Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2032516Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2132516Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2232516Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2332516Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2432516Sgibbs * SUCH DAMAGE. 2532516Sgibbs */ 2632516Sgibbs 27115683Sobrien#include <sys/cdefs.h> 28115683Sobrien__FBSDID("$FreeBSD: head/sys/i386/i386/busdma_machdep.c 169799 2007-05-20 16:53:45Z mjacob $"); 29115683Sobrien 3032516Sgibbs#include <sys/param.h> 31154367Sscottl#include <sys/kdb.h> 32154367Sscottl#include <ddb/ddb.h> 33154367Sscottl#include <ddb/db_output.h> 3432516Sgibbs#include <sys/systm.h> 3532516Sgibbs#include <sys/malloc.h> 3667551Sjhb#include <sys/bus.h> 3767551Sjhb#include <sys/interrupt.h> 38112346Smux#include <sys/kernel.h> 39136805Srwatson#include <sys/ktr.h> 4076827Salfred#include <sys/lock.h> 4179224Sdillon#include <sys/proc.h> 4276827Salfred#include <sys/mutex.h> 43104486Ssam#include <sys/mbuf.h> 44104486Ssam#include <sys/uio.h> 45131529Sscottl#include <sys/sysctl.h> 4632516Sgibbs 4732516Sgibbs#include <vm/vm.h> 4832516Sgibbs#include <vm/vm_page.h> 49104486Ssam#include <vm/vm_map.h> 5032516Sgibbs 51112436Smux#include <machine/atomic.h> 5232516Sgibbs#include <machine/bus.h> 5332516Sgibbs#include <machine/md_var.h> 54168822Sjhb#include <machine/specialreg.h> 5532516Sgibbs 56113228Sjake#define MAX_BPAGES 512 57162211Sscottl#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 58162211Sscottl#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 5932516Sgibbs 60137445Sscottlstruct bounce_zone; 61137445Sscottl 6232516Sgibbsstruct bus_dma_tag { 6332516Sgibbs bus_dma_tag_t parent; 6435767Sgibbs bus_size_t alignment; 6532516Sgibbs bus_size_t boundary; 6632516Sgibbs bus_addr_t lowaddr; 6732516Sgibbs bus_addr_t highaddr; 6832516Sgibbs bus_dma_filter_t *filter; 6932516Sgibbs void *filterarg; 7032516Sgibbs bus_size_t maxsize; 7135767Sgibbs u_int nsegments; 7232516Sgibbs bus_size_t maxsegsz; 7332516Sgibbs int flags; 7432516Sgibbs int ref_count; 7532516Sgibbs int map_count; 76117126Sscottl bus_dma_lock_t *lockfunc; 77117126Sscottl void *lockfuncarg; 78118246Sscottl bus_dma_segment_t *segments; 79137445Sscottl struct bounce_zone *bounce_zone; 8032516Sgibbs}; 8132516Sgibbs 82132545Sscottlstruct bounce_page { 83132545Sscottl vm_offset_t vaddr; /* kva of bounce buffer */ 84132545Sscottl bus_addr_t busaddr; /* Physical address */ 85132545Sscottl vm_offset_t datavaddr; /* kva of client data */ 86132545Sscottl bus_size_t datacount; /* client data count */ 87132545Sscottl STAILQ_ENTRY(bounce_page) links; 88132545Sscottl}; 89132545Sscottl 9032516Sgibbsint busdma_swi_pending; 9132516Sgibbs 92137445Sscottlstruct bounce_zone { 93137445Sscottl STAILQ_ENTRY(bounce_zone) links; 94137445Sscottl STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 95137965Sscottl int total_bpages; 96137445Sscottl int free_bpages; 97137445Sscottl int reserved_bpages; 98137445Sscottl int active_bpages; 99137445Sscottl int total_bounced; 100137445Sscottl int total_deferred; 101137445Sscottl bus_size_t alignment; 102137445Sscottl bus_size_t boundary; 103137445Sscottl bus_addr_t lowaddr; 104137445Sscottl char zoneid[8]; 105137445Sscottl char lowaddrid[20]; 106137445Sscottl struct sysctl_ctx_list sysctl_tree; 107137445Sscottl struct sysctl_oid *sysctl_tree_top; 108137445Sscottl}; 109137445Sscottl 110117136Smuxstatic struct mtx bounce_lock; 11132516Sgibbsstatic int total_bpages; 112137445Sscottlstatic int busdma_zonecount; 113137445Sscottlstatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 11432516Sgibbs 115131529SscottlSYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 116131529SscottlSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 117131529Sscottl "Total bounce pages"); 118131529Sscottl 11932516Sgibbsstruct bus_dmamap { 12032516Sgibbs struct bp_list bpages; 12132516Sgibbs int pagesneeded; 12232516Sgibbs int pagesreserved; 12332516Sgibbs bus_dma_tag_t dmat; 12432516Sgibbs void *buf; /* unmapped buffer pointer */ 12532516Sgibbs bus_size_t buflen; /* unmapped buffer length */ 12632516Sgibbs bus_dmamap_callback_t *callback; 12732516Sgibbs void *callback_arg; 12860938Sjake STAILQ_ENTRY(bus_dmamap) links; 12932516Sgibbs}; 13032516Sgibbs 13160938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 13260938Sjakestatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 13332516Sgibbsstatic struct bus_dmamap nobounce_dmamap; 13432516Sgibbs 135112346Smuxstatic void init_bounce_pages(void *dummy); 136137965Sscottlstatic int alloc_bounce_zone(bus_dma_tag_t dmat); 13732516Sgibbsstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 138113228Sjakestatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 139117136Smux int commit); 140112569Sjakestatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 14132516Sgibbs vm_offset_t vaddr, bus_size_t size); 14232516Sgibbsstatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 143162275Sscottlint run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 144162275Sscottlint _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 145162275Sscottl bus_size_t buflen, int flags, int *nb); 14632516Sgibbs 14795076Salfred/* 14895076Salfred * Return true if a match is made. 149117136Smux * 15095076Salfred * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 151117136Smux * 15295076Salfred * If paddr is within the bounds of the dma tag then call the filter callback 15395076Salfred * to check for a match, if there is no filter callback then assume a match. 15495076Salfred */ 155162275Sscottlint 156137894Sscottlrun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 15732516Sgibbs{ 15832516Sgibbs int retval; 15932516Sgibbs 16032516Sgibbs retval = 0; 161131529Sscottl 16232516Sgibbs do { 163131529Sscottl if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 164137894Sscottl || ((paddr & (dmat->alignment - 1)) != 0)) 16532516Sgibbs && (dmat->filter == NULL 166132545Sscottl || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 16732516Sgibbs retval = 1; 16832516Sgibbs 16932516Sgibbs dmat = dmat->parent; 17032516Sgibbs } while (retval == 0 && dmat != NULL); 17132516Sgibbs return (retval); 17232516Sgibbs} 17332516Sgibbs 174117126Sscottl/* 175117126Sscottl * Convenience function for manipulating driver locks from busdma (during 176117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 177117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 178117126Sscottl * non-mutex locking scheme don't have to use this at all. 179117126Sscottl */ 180117126Sscottlvoid 181117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 182117126Sscottl{ 183117126Sscottl struct mtx *dmtx; 184117126Sscottl 185117126Sscottl dmtx = (struct mtx *)arg; 186117126Sscottl switch (op) { 187117126Sscottl case BUS_DMA_LOCK: 188117126Sscottl mtx_lock(dmtx); 189117126Sscottl break; 190117126Sscottl case BUS_DMA_UNLOCK: 191117126Sscottl mtx_unlock(dmtx); 192117126Sscottl break; 193117126Sscottl default: 194117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 195117126Sscottl } 196117126Sscottl} 197117126Sscottl 198117126Sscottl/* 199117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 200117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 201117126Sscottl * with the tag are meant to never be defered. 202117126Sscottl * XXX Should have a way to identify which driver is responsible here. 203117126Sscottl */ 204117126Sscottlstatic void 205117126Sscottldflt_lock(void *arg, bus_dma_lock_op_t op) 206117126Sscottl{ 207117126Sscottl panic("driver error: busdma dflt_lock called"); 208117126Sscottl} 209117126Sscottl 21032516Sgibbs/* 21132516Sgibbs * Allocate a device specific dma_tag. 21232516Sgibbs */ 21332516Sgibbsint 21435767Sgibbsbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 21535767Sgibbs bus_size_t boundary, bus_addr_t lowaddr, 21635767Sgibbs bus_addr_t highaddr, bus_dma_filter_t *filter, 21735767Sgibbs void *filterarg, bus_size_t maxsize, int nsegments, 218117126Sscottl bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 219117126Sscottl void *lockfuncarg, bus_dma_tag_t *dmat) 22032516Sgibbs{ 22132516Sgibbs bus_dma_tag_t newtag; 22232516Sgibbs int error = 0; 22332516Sgibbs 224131529Sscottl /* Basic sanity checking */ 225131529Sscottl if (boundary != 0 && boundary < maxsegsz) 226131529Sscottl maxsegsz = boundary; 227131529Sscottl 22832516Sgibbs /* Return a NULL tag on failure */ 22932516Sgibbs *dmat = NULL; 23032516Sgibbs 231137460Sscottl newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 232137460Sscottl M_ZERO | M_NOWAIT); 233136805Srwatson if (newtag == NULL) { 234143293Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 235143284Smux __func__, newtag, 0, error); 23632516Sgibbs return (ENOMEM); 237136805Srwatson } 23832516Sgibbs 23932516Sgibbs newtag->parent = parent; 24048449Smjacob newtag->alignment = alignment; 24132516Sgibbs newtag->boundary = boundary; 242112569Sjake newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 243112569Sjake newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + 244112569Sjake (PAGE_SIZE - 1); 24532516Sgibbs newtag->filter = filter; 24632516Sgibbs newtag->filterarg = filterarg; 24732516Sgibbs newtag->maxsize = maxsize; 24832516Sgibbs newtag->nsegments = nsegments; 24932516Sgibbs newtag->maxsegsz = maxsegsz; 25032516Sgibbs newtag->flags = flags; 25132516Sgibbs newtag->ref_count = 1; /* Count ourself */ 25232516Sgibbs newtag->map_count = 0; 253117126Sscottl if (lockfunc != NULL) { 254117126Sscottl newtag->lockfunc = lockfunc; 255117126Sscottl newtag->lockfuncarg = lockfuncarg; 256117126Sscottl } else { 257117126Sscottl newtag->lockfunc = dflt_lock; 258117126Sscottl newtag->lockfuncarg = NULL; 259117126Sscottl } 260118246Sscottl newtag->segments = NULL; 261118246Sscottl 26232516Sgibbs /* Take into account any restrictions imposed by our parent tag */ 26332516Sgibbs if (parent != NULL) { 26432516Sgibbs newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 26532516Sgibbs newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 266134934Sscottl if (newtag->boundary == 0) 267134934Sscottl newtag->boundary = parent->boundary; 268134934Sscottl else if (parent->boundary != 0) 269134934Sscottl newtag->boundary = MIN(parent->boundary, 270134934Sscottl newtag->boundary); 271162211Sscottl if ((newtag->filter != NULL) || 272162673Sscottl ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 273162673Sscottl newtag->flags |= BUS_DMA_COULD_BOUNCE; 27432516Sgibbs if (newtag->filter == NULL) { 27532516Sgibbs /* 27632516Sgibbs * Short circuit looking at our parent directly 27735256Sdes * since we have encapsulated all of its information 27832516Sgibbs */ 27932516Sgibbs newtag->filter = parent->filter; 28032516Sgibbs newtag->filterarg = parent->filterarg; 28132516Sgibbs newtag->parent = parent->parent; 28232516Sgibbs } 283112436Smux if (newtag->parent != NULL) 284112436Smux atomic_add_int(&parent->ref_count, 1); 28532516Sgibbs } 286137965Sscottl 287137965Sscottl if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) 288138194Sscottl || newtag->alignment > 1) 289137965Sscottl newtag->flags |= BUS_DMA_COULD_BOUNCE; 290137965Sscottl 291137965Sscottl if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 292112569Sjake (flags & BUS_DMA_ALLOCNOW) != 0) { 293137965Sscottl struct bounce_zone *bz; 294137965Sscottl 29532516Sgibbs /* Must bounce */ 29632516Sgibbs 297154367Sscottl if ((error = alloc_bounce_zone(newtag)) != 0) { 298154367Sscottl free(newtag, M_DEVBUF); 299137965Sscottl return (error); 300154367Sscottl } 301137965Sscottl bz = newtag->bounce_zone; 302137965Sscottl 303137965Sscottl if (ptoa(bz->total_bpages) < maxsize) { 30432516Sgibbs int pages; 30532516Sgibbs 306137965Sscottl pages = atop(maxsize) - bz->total_bpages; 30732516Sgibbs 30832516Sgibbs /* Add pages to our bounce pool */ 30932516Sgibbs if (alloc_bounce_pages(newtag, pages) < pages) 31032516Sgibbs error = ENOMEM; 31132516Sgibbs } 31235767Sgibbs /* Performed initial allocation */ 31335767Sgibbs newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 31432516Sgibbs } 31532516Sgibbs 31632516Sgibbs if (error != 0) { 31732516Sgibbs free(newtag, M_DEVBUF); 31832516Sgibbs } else { 31932516Sgibbs *dmat = newtag; 32032516Sgibbs } 321143293Smux CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 322143284Smux __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 32332516Sgibbs return (error); 32432516Sgibbs} 32532516Sgibbs 32632516Sgibbsint 32732516Sgibbsbus_dma_tag_destroy(bus_dma_tag_t dmat) 32832516Sgibbs{ 329136805Srwatson bus_dma_tag_t dmat_copy; 330136805Srwatson int error; 331136805Srwatson 332136805Srwatson error = 0; 333136805Srwatson dmat_copy = dmat; 334136805Srwatson 33532516Sgibbs if (dmat != NULL) { 33632516Sgibbs 337136805Srwatson if (dmat->map_count != 0) { 338136805Srwatson error = EBUSY; 339136805Srwatson goto out; 340136805Srwatson } 34132516Sgibbs 34232516Sgibbs while (dmat != NULL) { 34332516Sgibbs bus_dma_tag_t parent; 34432516Sgibbs 34532516Sgibbs parent = dmat->parent; 346112436Smux atomic_subtract_int(&dmat->ref_count, 1); 34732516Sgibbs if (dmat->ref_count == 0) { 348118246Sscottl if (dmat->segments != NULL) 349118246Sscottl free(dmat->segments, M_DEVBUF); 35032516Sgibbs free(dmat, M_DEVBUF); 35140029Sgibbs /* 35240029Sgibbs * Last reference count, so 35340029Sgibbs * release our reference 35440029Sgibbs * count on our parent. 35540029Sgibbs */ 35640029Sgibbs dmat = parent; 35740029Sgibbs } else 35840029Sgibbs dmat = NULL; 35932516Sgibbs } 36032516Sgibbs } 361136805Srwatsonout: 362143293Smux CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 363136805Srwatson return (error); 36432516Sgibbs} 36532516Sgibbs 36632516Sgibbs/* 36732516Sgibbs * Allocate a handle for mapping from kva/uva/physical 36832516Sgibbs * address space into bus device space. 36932516Sgibbs */ 37032516Sgibbsint 37132516Sgibbsbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 37232516Sgibbs{ 37332516Sgibbs int error; 37432516Sgibbs 37532516Sgibbs error = 0; 37632516Sgibbs 377118246Sscottl if (dmat->segments == NULL) { 378118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 379118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 380118246Sscottl M_NOWAIT); 381136805Srwatson if (dmat->segments == NULL) { 382143293Smux CTR3(KTR_BUSDMA, "%s: tag %p error %d", 383143284Smux __func__, dmat, ENOMEM); 384118246Sscottl return (ENOMEM); 385136805Srwatson } 386118246Sscottl } 387118246Sscottl 388131529Sscottl /* 389131529Sscottl * Bouncing might be required if the driver asks for an active 390131529Sscottl * exclusion region, a data alignment that is stricter than 1, and/or 391131529Sscottl * an active address boundary. 392131529Sscottl */ 393137965Sscottl if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 394137445Sscottl 39532516Sgibbs /* Must bounce */ 396143449Sscottl struct bounce_zone *bz; 39732516Sgibbs int maxpages; 39832516Sgibbs 399137965Sscottl if (dmat->bounce_zone == NULL) { 400137965Sscottl if ((error = alloc_bounce_zone(dmat)) != 0) 401137965Sscottl return (error); 402137965Sscottl } 403143449Sscottl bz = dmat->bounce_zone; 404137965Sscottl 40532516Sgibbs *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 40669781Sdwmalone M_NOWAIT | M_ZERO); 407136805Srwatson if (*mapp == NULL) { 408143293Smux CTR3(KTR_BUSDMA, "%s: tag %p error %d", 409143284Smux __func__, dmat, ENOMEM); 41035767Sgibbs return (ENOMEM); 411136805Srwatson } 41269781Sdwmalone 41369781Sdwmalone /* Initialize the new map */ 41469781Sdwmalone STAILQ_INIT(&((*mapp)->bpages)); 41569781Sdwmalone 41632516Sgibbs /* 41732516Sgibbs * Attempt to add pages to our pool on a per-instance 41832516Sgibbs * basis up to a sane limit. 41932516Sgibbs */ 420143449Sscottl if (dmat->alignment > 1) 421143449Sscottl maxpages = MAX_BPAGES; 422143449Sscottl else 423143449Sscottl maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); 42435767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 425143449Sscottl || (dmat->map_count > 0 && bz->total_bpages < maxpages)) { 42632516Sgibbs int pages; 42732516Sgibbs 428113228Sjake pages = MAX(atop(dmat->maxsize), 1); 429143449Sscottl pages = MIN(maxpages - bz->total_bpages, pages); 430143449Sscottl pages = MAX(pages, 1); 431113228Sjake if (alloc_bounce_pages(dmat, pages) < pages) 432113228Sjake error = ENOMEM; 43335767Sgibbs 43435767Sgibbs if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 43535767Sgibbs if (error == 0) 43635767Sgibbs dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 43735767Sgibbs } else { 43835767Sgibbs error = 0; 43935767Sgibbs } 44032516Sgibbs } 44132516Sgibbs } else { 44240029Sgibbs *mapp = NULL; 44332516Sgibbs } 44432516Sgibbs if (error == 0) 44532516Sgibbs dmat->map_count++; 446143293Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 447143284Smux __func__, dmat, dmat->flags, error); 44832516Sgibbs return (error); 44932516Sgibbs} 45032516Sgibbs 45132516Sgibbs/* 45232516Sgibbs * Destroy a handle for mapping from kva/uva/physical 45332516Sgibbs * address space into bus device space. 45432516Sgibbs */ 45532516Sgibbsint 45632516Sgibbsbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 45732516Sgibbs{ 458117136Smux if (map != NULL && map != &nobounce_dmamap) { 459136805Srwatson if (STAILQ_FIRST(&map->bpages) != NULL) { 460143293Smux CTR3(KTR_BUSDMA, "%s: tag %p error %d", 461143284Smux __func__, dmat, EBUSY); 46232516Sgibbs return (EBUSY); 463136805Srwatson } 46432516Sgibbs free(map, M_DEVBUF); 46532516Sgibbs } 46632516Sgibbs dmat->map_count--; 467143293Smux CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 46832516Sgibbs return (0); 46932516Sgibbs} 47032516Sgibbs 47135767Sgibbs 47235767Sgibbs/* 47335767Sgibbs * Allocate a piece of memory that can be efficiently mapped into 47435767Sgibbs * bus device space based on the constraints lited in the dma tag. 47535767Sgibbs * A dmamap to for use with dmamap_load is also allocated. 47635767Sgibbs */ 47735767Sgibbsint 478115316Sscottlbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 479115316Sscottl bus_dmamap_t *mapp) 48035767Sgibbs{ 481159130Ssilby int mflags; 482118081Smux 483118081Smux if (flags & BUS_DMA_NOWAIT) 484118081Smux mflags = M_NOWAIT; 485118081Smux else 486118081Smux mflags = M_WAITOK; 487118081Smux if (flags & BUS_DMA_ZERO) 488118081Smux mflags |= M_ZERO; 489118081Smux 49035767Sgibbs /* If we succeed, no mapping/bouncing will be required */ 49140029Sgibbs *mapp = NULL; 49235767Sgibbs 493118246Sscottl if (dmat->segments == NULL) { 494118246Sscottl dmat->segments = (bus_dma_segment_t *)malloc( 495118246Sscottl sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 496118246Sscottl M_NOWAIT); 497136805Srwatson if (dmat->segments == NULL) { 498143293Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 499143284Smux __func__, dmat, dmat->flags, ENOMEM); 500118246Sscottl return (ENOMEM); 501136805Srwatson } 502118246Sscottl } 503118246Sscottl 504159011Ssilby /* 505159011Ssilby * XXX: 506159011Ssilby * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact 507159011Ssilby * alignment guarantees of malloc need to be nailed down, and the 508159011Ssilby * code below should be rewritten to take that into account. 509159011Ssilby * 510159130Ssilby * In the meantime, we'll warn the user if malloc gets it wrong. 511159011Ssilby */ 512115316Sscottl if ((dmat->maxsize <= PAGE_SIZE) && 513159011Ssilby (dmat->alignment < dmat->maxsize) && 514112569Sjake dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 515118081Smux *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 51635767Sgibbs } else { 51735767Sgibbs /* 51835767Sgibbs * XXX Use Contigmalloc until it is merged into this facility 51935767Sgibbs * and handles multi-seg allocations. Nobody is doing 52035767Sgibbs * multi-seg allocations yet though. 521131529Sscottl * XXX Certain AGP hardware does. 52235767Sgibbs */ 523118081Smux *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 52448449Smjacob 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 52548449Smjacob dmat->boundary); 52635767Sgibbs } 527136805Srwatson if (*vaddr == NULL) { 528143293Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 529143284Smux __func__, dmat, dmat->flags, ENOMEM); 53035767Sgibbs return (ENOMEM); 531159130Ssilby } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { 532162607Simp printf("bus_dmamem_alloc failed to align memory properly.\n"); 533159092Smjacob } 534168822Sjhb if (flags & BUS_DMA_NOCACHE) 535168822Sjhb pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize, 536168822Sjhb PAT_UNCACHEABLE); 537143293Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 538143284Smux __func__, dmat, dmat->flags, ENOMEM); 53935767Sgibbs return (0); 54035767Sgibbs} 54135767Sgibbs 54235767Sgibbs/* 54335767Sgibbs * Free a piece of memory and it's allociated dmamap, that was allocated 54495076Salfred * via bus_dmamem_alloc. Make the same choice for free/contigfree. 54535767Sgibbs */ 54635767Sgibbsvoid 547115316Sscottlbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 54835767Sgibbs{ 54935767Sgibbs /* 55035767Sgibbs * dmamem does not need to be bounced, so the map should be 55135767Sgibbs * NULL 55235767Sgibbs */ 55349859Sgibbs if (map != NULL) 55435767Sgibbs panic("bus_dmamem_free: Invalid map freed\n"); 555168822Sjhb pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK); 556159011Ssilby if ((dmat->maxsize <= PAGE_SIZE) && 557159011Ssilby (dmat->alignment < dmat->maxsize) && 558159011Ssilby dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 55940029Sgibbs free(vaddr, M_DEVBUF); 560112196Smux else { 561115316Sscottl contigfree(vaddr, dmat->maxsize, M_DEVBUF); 562112196Smux } 563143293Smux CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 56435767Sgibbs} 56535767Sgibbs 566162275Sscottlint 567162211Sscottl_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 568162211Sscottl bus_size_t buflen, int flags, int *nb) 569104486Ssam{ 570113228Sjake vm_offset_t vaddr; 571162211Sscottl vm_offset_t vendaddr; 572113228Sjake bus_addr_t paddr; 573162211Sscottl int needbounce = *nb; 574104486Ssam 575162211Sscottl if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { 576137142Sscottl CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 577137142Sscottl "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), 578137142Sscottl dmat->boundary, dmat->alignment); 579137142Sscottl CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", 580137142Sscottl map, &nobounce_dmamap, map->pagesneeded); 581113228Sjake /* 582113228Sjake * Count the number of bounce pages 583113228Sjake * needed in order to complete this transfer 584113228Sjake */ 585113228Sjake vaddr = trunc_page((vm_offset_t)buf); 586113228Sjake vendaddr = (vm_offset_t)buf + buflen; 587113228Sjake 588113228Sjake while (vaddr < vendaddr) { 589113228Sjake paddr = pmap_kextract(vaddr); 590162673Sscottl if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 591162211Sscottl run_filter(dmat, paddr) != 0) { 592113228Sjake needbounce = 1; 593113228Sjake map->pagesneeded++; 594113228Sjake } 595113228Sjake vaddr += PAGE_SIZE; 596113228Sjake } 597137142Sscottl CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 598113228Sjake } 599113228Sjake 600113228Sjake /* Reserve Necessary Bounce Pages */ 601113228Sjake if (map->pagesneeded != 0) { 602113228Sjake mtx_lock(&bounce_lock); 603113472Ssimokawa if (flags & BUS_DMA_NOWAIT) { 604113472Ssimokawa if (reserve_bounce_pages(dmat, map, 0) != 0) { 605113472Ssimokawa mtx_unlock(&bounce_lock); 606113472Ssimokawa return (ENOMEM); 607113472Ssimokawa } 608113472Ssimokawa } else { 609113472Ssimokawa if (reserve_bounce_pages(dmat, map, 1) != 0) { 610132545Sscottl /* Queue us for resources */ 611113472Ssimokawa map->dmat = dmat; 612113472Ssimokawa map->buf = buf; 613113472Ssimokawa map->buflen = buflen; 614113472Ssimokawa STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 615117136Smux map, links); 616113472Ssimokawa mtx_unlock(&bounce_lock); 617113472Ssimokawa return (EINPROGRESS); 618113472Ssimokawa } 619113228Sjake } 620113228Sjake mtx_unlock(&bounce_lock); 621113228Sjake } 622113228Sjake 623162211Sscottl *nb = needbounce; 624162211Sscottl return (0); 625162211Sscottl} 626162211Sscottl 627162211Sscottl/* 628162211Sscottl * Utility function to load a linear buffer. lastaddrp holds state 629162211Sscottl * between invocations (for multiple-buffer loads). segp contains 630162211Sscottl * the starting segment on entrace, and the ending segment on exit. 631162211Sscottl * first indicates if this is the first invocation of this function. 632162211Sscottl */ 633162211Sscottlstatic __inline int 634162211Sscottl_bus_dmamap_load_buffer(bus_dma_tag_t dmat, 635162211Sscottl bus_dmamap_t map, 636162211Sscottl void *buf, bus_size_t buflen, 637162211Sscottl pmap_t pmap, 638162211Sscottl int flags, 639162211Sscottl bus_addr_t *lastaddrp, 640162211Sscottl bus_dma_segment_t *segs, 641162211Sscottl int *segp, 642162211Sscottl int first) 643162211Sscottl{ 644162211Sscottl bus_size_t sgsize; 645162211Sscottl bus_addr_t curaddr, lastaddr, baddr, bmask; 646162211Sscottl vm_offset_t vaddr; 647162211Sscottl int needbounce = 0; 648162211Sscottl int seg, error; 649162211Sscottl 650162211Sscottl if (map == NULL) 651162211Sscottl map = &nobounce_dmamap; 652162211Sscottl 653162211Sscottl if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 654162211Sscottl error = _bus_dmamap_count_pages(dmat, map, buf, buflen, flags, 655162211Sscottl &needbounce); 656162211Sscottl if (error) 657162211Sscottl return (error); 658162211Sscottl } 659162211Sscottl 660137142Sscottl vaddr = (vm_offset_t)buf; 661104486Ssam lastaddr = *lastaddrp; 662113228Sjake bmask = ~(dmat->boundary - 1); 663104486Ssam 664104486Ssam for (seg = *segp; buflen > 0 ; ) { 665104486Ssam /* 666104486Ssam * Get the physical address for this segment. 667104486Ssam */ 668104486Ssam if (pmap) 669104486Ssam curaddr = pmap_extract(pmap, vaddr); 670104486Ssam else 671104486Ssam curaddr = pmap_kextract(vaddr); 672104486Ssam 673104486Ssam /* 674104486Ssam * Compute the segment size, and adjust counts. 675104486Ssam */ 676104486Ssam sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 677104486Ssam if (buflen < sgsize) 678104486Ssam sgsize = buflen; 679104486Ssam 680104486Ssam /* 681104486Ssam * Make sure we don't cross any boundaries. 682104486Ssam */ 683104486Ssam if (dmat->boundary > 0) { 684104486Ssam baddr = (curaddr + dmat->boundary) & bmask; 685104486Ssam if (sgsize > (baddr - curaddr)) 686104486Ssam sgsize = (baddr - curaddr); 687104486Ssam } 688104486Ssam 689162673Sscottl if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 690162211Sscottl map->pagesneeded != 0 && run_filter(dmat, curaddr)) 691113228Sjake curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 692113228Sjake 693104486Ssam /* 694104486Ssam * Insert chunk into a segment, coalescing with 695104486Ssam * previous segment if possible. 696104486Ssam */ 697104486Ssam if (first) { 698104486Ssam segs[seg].ds_addr = curaddr; 699104486Ssam segs[seg].ds_len = sgsize; 700104486Ssam first = 0; 701104486Ssam } else { 702113228Sjake if (needbounce == 0 && curaddr == lastaddr && 703104486Ssam (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 704104486Ssam (dmat->boundary == 0 || 705104486Ssam (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 706104486Ssam segs[seg].ds_len += sgsize; 707104486Ssam else { 708104486Ssam if (++seg >= dmat->nsegments) 709104486Ssam break; 710104486Ssam segs[seg].ds_addr = curaddr; 711104486Ssam segs[seg].ds_len = sgsize; 712104486Ssam } 713104486Ssam } 714104486Ssam 715104486Ssam lastaddr = curaddr + sgsize; 716104486Ssam vaddr += sgsize; 717104486Ssam buflen -= sgsize; 718104486Ssam } 719104486Ssam 720104486Ssam *segp = seg; 721104486Ssam *lastaddrp = lastaddr; 722104486Ssam 723104486Ssam /* 724104486Ssam * Did we fit? 725104486Ssam */ 726104486Ssam return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 727104486Ssam} 728104486Ssam 729104486Ssam/* 730113459Ssimokawa * Map the buffer buf into bus space using the dmamap map. 731113459Ssimokawa */ 732113459Ssimokawaint 733113459Ssimokawabus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 734113459Ssimokawa bus_size_t buflen, bus_dmamap_callback_t *callback, 735113459Ssimokawa void *callback_arg, int flags) 736113459Ssimokawa{ 737113492Smux bus_addr_t lastaddr = 0; 738113459Ssimokawa int error, nsegs = 0; 739113459Ssimokawa 740113472Ssimokawa if (map != NULL) { 741113472Ssimokawa flags |= BUS_DMA_WAITOK; 742113472Ssimokawa map->callback = callback; 743113472Ssimokawa map->callback_arg = callback_arg; 744113472Ssimokawa } 745113472Ssimokawa 746118246Sscottl error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 747139840Sscottl &lastaddr, dmat->segments, &nsegs, 1); 748113459Ssimokawa 749158264Sscottl CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 750158264Sscottl __func__, dmat, dmat->flags, error, nsegs + 1); 751158264Sscottl 752136805Srwatson if (error == EINPROGRESS) { 753113492Smux return (error); 754136805Srwatson } 755113472Ssimokawa 756113459Ssimokawa if (error) 757118246Sscottl (*callback)(callback_arg, dmat->segments, 0, error); 758113459Ssimokawa else 759118246Sscottl (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 760113459Ssimokawa 761158264Sscottl /* 762158264Sscottl * Return ENOMEM to the caller so that it can pass it up the stack. 763158264Sscottl * This error only happens when NOWAIT is set, so deferal is disabled. 764158264Sscottl */ 765158264Sscottl if (error == ENOMEM) 766158264Sscottl return (error); 767158264Sscottl 768113459Ssimokawa return (0); 769113459Ssimokawa} 770113459Ssimokawa 771113459Ssimokawa 772113459Ssimokawa/* 773104486Ssam * Like _bus_dmamap_load(), but for mbufs. 774104486Ssam */ 775162275Sscottlstatic __inline int 776162275Sscottl_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 777162275Sscottl struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 778162275Sscottl int flags) 779104486Ssam{ 780162275Sscottl int error; 781104486Ssam 782117136Smux M_ASSERTPKTHDR(m0); 783104486Ssam 784113472Ssimokawa flags |= BUS_DMA_NOWAIT; 785162275Sscottl *nsegs = 0; 786104486Ssam error = 0; 787104486Ssam if (m0->m_pkthdr.len <= dmat->maxsize) { 788104486Ssam int first = 1; 789113228Sjake bus_addr_t lastaddr = 0; 790104486Ssam struct mbuf *m; 791104486Ssam 792104486Ssam for (m = m0; m != NULL && error == 0; m = m->m_next) { 793110335Sharti if (m->m_len > 0) { 794113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 795110335Sharti m->m_data, m->m_len, 796110335Sharti NULL, flags, &lastaddr, 797162275Sscottl segs, nsegs, first); 798110335Sharti first = 0; 799110335Sharti } 800104486Ssam } 801104486Ssam } else { 802104486Ssam error = EINVAL; 803104486Ssam } 804104486Ssam 805162275Sscottl /* XXX FIXME: Having to increment nsegs is really annoying */ 806162275Sscottl ++*nsegs; 807162275Sscottl CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 808162275Sscottl __func__, dmat, dmat->flags, error, *nsegs); 809162275Sscottl return (error); 810162275Sscottl} 811162275Sscottl 812162275Sscottlint 813162275Sscottlbus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 814162275Sscottl struct mbuf *m0, 815162275Sscottl bus_dmamap_callback2_t *callback, void *callback_arg, 816162275Sscottl int flags) 817162275Sscottl{ 818162275Sscottl int nsegs, error; 819162275Sscottl 820162275Sscottl error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, 821162275Sscottl flags); 822162275Sscottl 823104486Ssam if (error) { 824104486Ssam /* force "no valid mappings" in callback */ 825118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 826104486Ssam } else { 827118246Sscottl (*callback)(callback_arg, dmat->segments, 828162275Sscottl nsegs, m0->m_pkthdr.len, error); 829104486Ssam } 830143293Smux CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 831162275Sscottl __func__, dmat, dmat->flags, error, nsegs); 832104486Ssam return (error); 833104486Ssam} 834104486Ssam 835139840Sscottlint 836139840Sscottlbus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 837139840Sscottl struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 838139840Sscottl int flags) 839139840Sscottl{ 840162275Sscottl return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); 841139840Sscottl} 842139840Sscottl 843104486Ssam/* 844104486Ssam * Like _bus_dmamap_load(), but for uios. 845104486Ssam */ 846104486Ssamint 847104486Ssambus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 848104486Ssam struct uio *uio, 849104486Ssam bus_dmamap_callback2_t *callback, void *callback_arg, 850104486Ssam int flags) 851104486Ssam{ 852113228Sjake bus_addr_t lastaddr; 853104486Ssam int nsegs, error, first, i; 854104486Ssam bus_size_t resid; 855104486Ssam struct iovec *iov; 856137142Sscottl pmap_t pmap; 857104486Ssam 858113472Ssimokawa flags |= BUS_DMA_NOWAIT; 859104486Ssam resid = uio->uio_resid; 860104486Ssam iov = uio->uio_iov; 861104486Ssam 862104486Ssam if (uio->uio_segflg == UIO_USERSPACE) { 863137142Sscottl KASSERT(uio->uio_td != NULL, 864104486Ssam ("bus_dmamap_load_uio: USERSPACE but no proc")); 865137142Sscottl pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 866137142Sscottl } else 867137142Sscottl pmap = NULL; 868104486Ssam 869104486Ssam nsegs = 0; 870104486Ssam error = 0; 871104486Ssam first = 1; 872169799Smjacob lastaddr = (bus_addr_t) 0; 873104486Ssam for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 874104486Ssam /* 875104486Ssam * Now at the first iovec to load. Load each iovec 876104486Ssam * until we have exhausted the residual count. 877104486Ssam */ 878104486Ssam bus_size_t minlen = 879104486Ssam resid < iov[i].iov_len ? resid : iov[i].iov_len; 880104486Ssam caddr_t addr = (caddr_t) iov[i].iov_base; 881104486Ssam 882110335Sharti if (minlen > 0) { 883113228Sjake error = _bus_dmamap_load_buffer(dmat, map, 884139840Sscottl addr, minlen, pmap, flags, &lastaddr, 885139840Sscottl dmat->segments, &nsegs, first); 886110335Sharti first = 0; 887104486Ssam 888110335Sharti resid -= minlen; 889110335Sharti } 890104486Ssam } 891104486Ssam 892104486Ssam if (error) { 893104486Ssam /* force "no valid mappings" in callback */ 894118246Sscottl (*callback)(callback_arg, dmat->segments, 0, 0, error); 895104486Ssam } else { 896118246Sscottl (*callback)(callback_arg, dmat->segments, 897104486Ssam nsegs+1, uio->uio_resid, error); 898104486Ssam } 899143293Smux CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 900143284Smux __func__, dmat, dmat->flags, error, nsegs + 1); 901104486Ssam return (error); 902104486Ssam} 903104486Ssam 904104486Ssam/* 90532516Sgibbs * Release the mapping held by map. 90632516Sgibbs */ 90732516Sgibbsvoid 90832516Sgibbs_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 90932516Sgibbs{ 91032516Sgibbs struct bounce_page *bpage; 91132516Sgibbs 91232516Sgibbs while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 91332516Sgibbs STAILQ_REMOVE_HEAD(&map->bpages, links); 91432516Sgibbs free_bounce_page(dmat, bpage); 91532516Sgibbs } 91632516Sgibbs} 91732516Sgibbs 91832516Sgibbsvoid 919115343Sscottl_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 92032516Sgibbs{ 92132516Sgibbs struct bounce_page *bpage; 92232516Sgibbs 92332516Sgibbs if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 92432516Sgibbs /* 92532516Sgibbs * Handle data bouncing. We might also 92632516Sgibbs * want to add support for invalidating 92732516Sgibbs * the caches on broken hardware 92832516Sgibbs */ 929143293Smux CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 930143284Smux "performing bounce", __func__, op, dmat, dmat->flags); 931131529Sscottl 932113347Smux if (op & BUS_DMASYNC_PREWRITE) { 93332516Sgibbs while (bpage != NULL) { 93432516Sgibbs bcopy((void *)bpage->datavaddr, 93532516Sgibbs (void *)bpage->vaddr, 93632516Sgibbs bpage->datacount); 93732516Sgibbs bpage = STAILQ_NEXT(bpage, links); 93832516Sgibbs } 939167277Sscottl dmat->bounce_zone->total_bounced++; 940113347Smux } 94132516Sgibbs 942113347Smux if (op & BUS_DMASYNC_POSTREAD) { 94332516Sgibbs while (bpage != NULL) { 94432516Sgibbs bcopy((void *)bpage->vaddr, 94532516Sgibbs (void *)bpage->datavaddr, 94632516Sgibbs bpage->datacount); 94732516Sgibbs bpage = STAILQ_NEXT(bpage, links); 94832516Sgibbs } 949167277Sscottl dmat->bounce_zone->total_bounced++; 95032516Sgibbs } 95132516Sgibbs } 95232516Sgibbs} 95332516Sgibbs 954112346Smuxstatic void 955112346Smuxinit_bounce_pages(void *dummy __unused) 956112346Smux{ 957112346Smux 958112346Smux total_bpages = 0; 959137445Sscottl STAILQ_INIT(&bounce_zone_list); 960112346Smux STAILQ_INIT(&bounce_map_waitinglist); 961112346Smux STAILQ_INIT(&bounce_map_callbacklist); 962112346Smux mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 963112346Smux} 964112346SmuxSYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 965112346Smux 966137445Sscottlstatic struct sysctl_ctx_list * 967137445Sscottlbusdma_sysctl_tree(struct bounce_zone *bz) 968137445Sscottl{ 969137445Sscottl return (&bz->sysctl_tree); 970137445Sscottl} 971137445Sscottl 972137445Sscottlstatic struct sysctl_oid * 973137445Sscottlbusdma_sysctl_tree_top(struct bounce_zone *bz) 974137445Sscottl{ 975137445Sscottl return (bz->sysctl_tree_top); 976137445Sscottl} 977137445Sscottl 978137965Sscottlstatic int 979137445Sscottlalloc_bounce_zone(bus_dma_tag_t dmat) 980137445Sscottl{ 981137445Sscottl struct bounce_zone *bz; 982137445Sscottl 983137965Sscottl /* Check to see if we already have a suitable zone */ 984137965Sscottl STAILQ_FOREACH(bz, &bounce_zone_list, links) { 985137965Sscottl if ((dmat->alignment <= bz->alignment) 986137965Sscottl && (dmat->boundary <= bz->boundary) 987137965Sscottl && (dmat->lowaddr >= bz->lowaddr)) { 988137965Sscottl dmat->bounce_zone = bz; 989137965Sscottl return (0); 990137965Sscottl } 991137965Sscottl } 992137965Sscottl 993137445Sscottl if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 994137445Sscottl M_NOWAIT | M_ZERO)) == NULL) 995137965Sscottl return (ENOMEM); 996137445Sscottl 997137445Sscottl STAILQ_INIT(&bz->bounce_page_list); 998137445Sscottl bz->free_bpages = 0; 999137445Sscottl bz->reserved_bpages = 0; 1000137445Sscottl bz->active_bpages = 0; 1001137445Sscottl bz->lowaddr = dmat->lowaddr; 1002137445Sscottl bz->alignment = dmat->alignment; 1003137445Sscottl bz->boundary = dmat->boundary; 1004137445Sscottl snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1005137445Sscottl busdma_zonecount++; 1006137460Sscottl snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1007137445Sscottl STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1008137965Sscottl dmat->bounce_zone = bz; 1009137445Sscottl 1010137445Sscottl sysctl_ctx_init(&bz->sysctl_tree); 1011137445Sscottl bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1012137445Sscottl SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1013137445Sscottl CTLFLAG_RD, 0, ""); 1014137445Sscottl if (bz->sysctl_tree_top == NULL) { 1015137445Sscottl sysctl_ctx_free(&bz->sysctl_tree); 1016137965Sscottl return (0); /* XXX error code? */ 1017137445Sscottl } 1018137445Sscottl 1019137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1020137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1021137965Sscottl "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1022152775Sle "Total bounce pages"); 1023137965Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1024137965Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1025137445Sscottl "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1026137445Sscottl "Free bounce pages"); 1027137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1028137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1029137445Sscottl "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1030137445Sscottl "Reserved bounce pages"); 1031137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1032137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1033137445Sscottl "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1034137445Sscottl "Active bounce pages"); 1035137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1036137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1037137445Sscottl "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1038137445Sscottl "Total bounce requests"); 1039137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1040137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1041137445Sscottl "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1042137445Sscottl "Total bounce requests that were deferred"); 1043137445Sscottl SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1044137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1045137445Sscottl "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1046137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1047137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1048137445Sscottl "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1049137445Sscottl SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1050137445Sscottl SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1051137445Sscottl "boundary", CTLFLAG_RD, &bz->boundary, 0, ""); 1052137445Sscottl 1053137965Sscottl return (0); 1054137445Sscottl} 1055137445Sscottl 105632516Sgibbsstatic int 105732516Sgibbsalloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 105832516Sgibbs{ 1059137445Sscottl struct bounce_zone *bz; 106032516Sgibbs int count; 106132516Sgibbs 1062137445Sscottl bz = dmat->bounce_zone; 106332516Sgibbs count = 0; 106432516Sgibbs while (numpages > 0) { 106532516Sgibbs struct bounce_page *bpage; 106632516Sgibbs 106732516Sgibbs bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 106869781Sdwmalone M_NOWAIT | M_ZERO); 106932516Sgibbs 107032516Sgibbs if (bpage == NULL) 107132516Sgibbs break; 107232516Sgibbs bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 107332516Sgibbs M_NOWAIT, 0ul, 1074137445Sscottl bz->lowaddr, 1075132545Sscottl PAGE_SIZE, 1076137445Sscottl bz->boundary); 1077102241Sarchie if (bpage->vaddr == 0) { 107832516Sgibbs free(bpage, M_DEVBUF); 107932516Sgibbs break; 108032516Sgibbs } 108132516Sgibbs bpage->busaddr = pmap_kextract(bpage->vaddr); 1082112346Smux mtx_lock(&bounce_lock); 1083137445Sscottl STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 108432516Sgibbs total_bpages++; 1085137965Sscottl bz->total_bpages++; 1086137445Sscottl bz->free_bpages++; 1087112346Smux mtx_unlock(&bounce_lock); 108832516Sgibbs count++; 108932516Sgibbs numpages--; 109032516Sgibbs } 109132516Sgibbs return (count); 109232516Sgibbs} 109332516Sgibbs 109432516Sgibbsstatic int 1095113228Sjakereserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 109632516Sgibbs{ 1097137445Sscottl struct bounce_zone *bz; 109832516Sgibbs int pages; 109932516Sgibbs 1100112346Smux mtx_assert(&bounce_lock, MA_OWNED); 1101137445Sscottl bz = dmat->bounce_zone; 1102137445Sscottl pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1103113228Sjake if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1104113228Sjake return (map->pagesneeded - (map->pagesreserved + pages)); 1105137445Sscottl bz->free_bpages -= pages; 1106137445Sscottl bz->reserved_bpages += pages; 110732516Sgibbs map->pagesreserved += pages; 110832516Sgibbs pages = map->pagesneeded - map->pagesreserved; 110932516Sgibbs 111032516Sgibbs return (pages); 111132516Sgibbs} 111232516Sgibbs 1113112569Sjakestatic bus_addr_t 111432516Sgibbsadd_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 111532516Sgibbs bus_size_t size) 111632516Sgibbs{ 1117137445Sscottl struct bounce_zone *bz; 111832516Sgibbs struct bounce_page *bpage; 111932516Sgibbs 1120137445Sscottl KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1121113228Sjake KASSERT(map != NULL && map != &nobounce_dmamap, 1122113228Sjake ("add_bounce_page: bad map %p", map)); 1123113228Sjake 1124137445Sscottl bz = dmat->bounce_zone; 112532516Sgibbs if (map->pagesneeded == 0) 112632516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 112732516Sgibbs map->pagesneeded--; 112832516Sgibbs 112932516Sgibbs if (map->pagesreserved == 0) 113032516Sgibbs panic("add_bounce_page: map doesn't need any pages"); 113132516Sgibbs map->pagesreserved--; 113232516Sgibbs 1133112346Smux mtx_lock(&bounce_lock); 1134137445Sscottl bpage = STAILQ_FIRST(&bz->bounce_page_list); 113532516Sgibbs if (bpage == NULL) 113632516Sgibbs panic("add_bounce_page: free page list is empty"); 113732516Sgibbs 1138137445Sscottl STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1139137445Sscottl bz->reserved_bpages--; 1140137445Sscottl bz->active_bpages++; 1141112346Smux mtx_unlock(&bounce_lock); 114232516Sgibbs 114332516Sgibbs bpage->datavaddr = vaddr; 114432516Sgibbs bpage->datacount = size; 114532516Sgibbs STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 114632516Sgibbs return (bpage->busaddr); 114732516Sgibbs} 114832516Sgibbs 114932516Sgibbsstatic void 115032516Sgibbsfree_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 115132516Sgibbs{ 115232516Sgibbs struct bus_dmamap *map; 1153137445Sscottl struct bounce_zone *bz; 115432516Sgibbs 1155137445Sscottl bz = dmat->bounce_zone; 115632516Sgibbs bpage->datavaddr = 0; 115732516Sgibbs bpage->datacount = 0; 115832516Sgibbs 1159112346Smux mtx_lock(&bounce_lock); 1160137445Sscottl STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1161137445Sscottl bz->free_bpages++; 1162137445Sscottl bz->active_bpages--; 116332516Sgibbs if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1164113228Sjake if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 116532516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 116632516Sgibbs STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 116732516Sgibbs map, links); 116832516Sgibbs busdma_swi_pending = 1; 1169137445Sscottl bz->total_deferred++; 117088900Sjhb swi_sched(vm_ih, 0); 117132516Sgibbs } 117232516Sgibbs } 1173112346Smux mtx_unlock(&bounce_lock); 117432516Sgibbs} 117532516Sgibbs 117632516Sgibbsvoid 117795076Salfredbusdma_swi(void) 117832516Sgibbs{ 1179117126Sscottl bus_dma_tag_t dmat; 118032516Sgibbs struct bus_dmamap *map; 118132516Sgibbs 1182112346Smux mtx_lock(&bounce_lock); 118332516Sgibbs while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 118432516Sgibbs STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1185112346Smux mtx_unlock(&bounce_lock); 1186117136Smux dmat = map->dmat; 1187117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 118832516Sgibbs bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 118932516Sgibbs map->callback, map->callback_arg, /*flags*/0); 1190117126Sscottl (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1191112346Smux mtx_lock(&bounce_lock); 119232516Sgibbs } 1193112346Smux mtx_unlock(&bounce_lock); 119432516Sgibbs} 1195