busdma_machdep-v4.c revision 244473
1139749Simp/*- 2113584Ssimokawa * Copyright (c) 2012 Ian Lepore 3103285Sikob * Copyright (c) 2004 Olivier Houchard 4103285Sikob * Copyright (c) 2002 Peter Grehan 5103285Sikob * Copyright (c) 1997, 1998 Justin T. Gibbs. 6103285Sikob * All rights reserved. 7103285Sikob * 8103285Sikob * Redistribution and use in source and binary forms, with or without 9103285Sikob * modification, are permitted provided that the following conditions 10103285Sikob * are met: 11103285Sikob * 1. Redistributions of source code must retain the above copyright 12103285Sikob * notice, this list of conditions, and the following disclaimer, 13103285Sikob * without modification, immediately at the beginning of the file. 14103285Sikob * 2. The name of the author may not be used to endorse or promote products 15103285Sikob * derived from this software without specific prior written permission. 16103285Sikob * 17103285Sikob * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18103285Sikob * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19103285Sikob * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20103285Sikob * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 21103285Sikob * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22103285Sikob * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23103285Sikob * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24103285Sikob * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25103285Sikob * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26103285Sikob * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27103285Sikob * SUCH DAMAGE. 28103285Sikob * 29103285Sikob * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 30103285Sikob */ 31103285Sikob 32103285Sikob#include <sys/cdefs.h> 33103285Sikob__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 244473 2012-12-20 00:50:04Z cognet $"); 34103285Sikob 35103285Sikob/* 36127468Ssimokawa * ARM bus dma support routines. 37119418Sobrien * 38119418Sobrien * XXX Things to investigate / fix some day... 39127468Ssimokawa * - What is the earliest that this API can be called? Could there be any 40119418Sobrien * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? 41103285Sikob * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the 42103285Sikob * bus_dmamap_load() function. This code has historically (and still does) 43103285Sikob * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some 44103285Sikob * error checking because some resource management calls would become WAITOK 45103285Sikob * and thus "cannot fail." 46103285Sikob * - The decisions made by _bus_dma_can_bounce() should be made once, at tag 47103285Sikob * creation time, and the result stored in the tag. 48103285Sikob * - It should be possible to take some shortcuts when mapping a buffer we know 49127468Ssimokawa * came from the uma(9) allocators based on what we know about such buffers 50120660Ssimokawa * (aligned, contiguous, etc). 51120660Ssimokawa * - The allocation of bounce pages could probably be cleaned up, then we could 52120660Ssimokawa * retire arm_remap_nocache(). 53120660Ssimokawa */ 54103285Sikob 55103285Sikob#define _ARM32_BUS_DMA_PRIVATE 56113584Ssimokawa#include <sys/param.h> 57103285Sikob#include <sys/systm.h> 58103285Sikob#include <sys/malloc.h> 59103285Sikob#include <sys/bus.h> 60103285Sikob#include <sys/busdma_bufalloc.h> 61122228Ssimokawa#include <sys/interrupt.h> 62103285Sikob#include <sys/lock.h> 63127468Ssimokawa#include <sys/proc.h> 64127468Ssimokawa#include <sys/mutex.h> 65127468Ssimokawa#include <sys/mbuf.h> 66127468Ssimokawa#include <sys/uio.h> 67127468Ssimokawa#include <sys/ktr.h> 68103285Sikob#include <sys/kernel.h> 69103285Sikob#include <sys/sysctl.h> 70103285Sikob 71127468Ssimokawa#include <vm/uma.h> 72103285Sikob#include <vm/vm.h> 73106810Ssimokawa#include <vm/vm_extern.h> 74106810Ssimokawa#include <vm/vm_kern.h> 75103285Sikob#include <vm/vm_page.h> 76103285Sikob#include <vm/vm_map.h> 77108281Ssimokawa 78106810Ssimokawa#include <machine/atomic.h> 79106810Ssimokawa#include <machine/bus.h> 80110582Ssimokawa#include <machine/cpufunc.h> 81106810Ssimokawa#include <machine/md_var.h> 82103285Sikob 83103285Sikob#define MAX_BPAGES 64 84103285Sikob#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 85103285Sikob#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 86103285Sikob 87124145Ssimokawastruct bounce_zone; 88124145Ssimokawa 89120660Ssimokawastruct bus_dma_tag { 90120660Ssimokawa bus_dma_tag_t parent; 91122228Ssimokawa bus_size_t alignment; 92122228Ssimokawa bus_addr_t boundary; 93122228Ssimokawa bus_addr_t lowaddr; 94122228Ssimokawa bus_addr_t highaddr; 95122228Ssimokawa bus_dma_filter_t *filter; 96106816Ssimokawa void *filterarg; 97106816Ssimokawa bus_size_t maxsize; 98106810Ssimokawa u_int nsegments; 99106816Ssimokawa bus_size_t maxsegsz; 100106816Ssimokawa int flags; 101113584Ssimokawa int ref_count; 102113584Ssimokawa int map_count; 103106816Ssimokawa bus_dma_lock_t *lockfunc; 104103285Sikob void *lockfuncarg; 105103285Sikob /* 106103285Sikob * DMA range for this tag. If the page doesn't fall within 107124145Ssimokawa * one of these ranges, an error is returned. The caller 108106804Ssimokawa * may then decide what to do with the transfer. If the 109103285Sikob * range pointer is NULL, it is ignored. 110106804Ssimokawa */ 111106810Ssimokawa struct arm32_dma_range *ranges; 112120660Ssimokawa int _nranges; 113106816Ssimokawa struct bounce_zone *bounce_zone; 114120660Ssimokawa /* 115106816Ssimokawa * Most tags need one or two segments, and can use the local tagsegs 116120660Ssimokawa * array. For tags with a larger limit, we'll allocate a bigger array 117106804Ssimokawa * on first use. 118103285Sikob */ 119106816Ssimokawa bus_dma_segment_t *segments; 120120660Ssimokawa bus_dma_segment_t tagsegs[2]; 121120660Ssimokawa}; 122103285Sikob 123106816Ssimokawastruct bounce_page { 124106816Ssimokawa vm_offset_t vaddr; /* kva of bounce buffer */ 125106816Ssimokawa vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */ 126106816Ssimokawa bus_addr_t busaddr; /* Physical address */ 127106816Ssimokawa vm_offset_t datavaddr; /* kva of client data */ 128106816Ssimokawa bus_size_t datacount; /* client data count */ 129106816Ssimokawa STAILQ_ENTRY(bounce_page) links; 130129585Sdfr}; 131129585Sdfr 132129585Sdfrint busdma_swi_pending; 133120660Ssimokawa 134106816Ssimokawastruct bounce_zone { 135106816Ssimokawa STAILQ_ENTRY(bounce_zone) links; 136106816Ssimokawa STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 137106816Ssimokawa int total_bpages; 138106816Ssimokawa int free_bpages; 139120660Ssimokawa int reserved_bpages; 140120660Ssimokawa int active_bpages; 141106816Ssimokawa int total_bounced; 142120660Ssimokawa int total_deferred; 143106816Ssimokawa int map_count; 144120660Ssimokawa bus_size_t alignment; 145103285Sikob bus_addr_t lowaddr; 146113584Ssimokawa char zoneid[8]; 147113584Ssimokawa char lowaddrid[20]; 148103285Sikob struct sysctl_ctx_list sysctl_tree; 149120660Ssimokawa struct sysctl_oid *sysctl_tree_top; 150129585Sdfr}; 151120660Ssimokawa 152103285Sikobstatic struct mtx bounce_lock; 153106816Ssimokawastatic int total_bpages; 154106816Ssimokawastatic int busdma_zonecount; 155106804Ssimokawastatic STAILQ_HEAD(, bounce_zone) bounce_zone_list; 156106810Ssimokawa 157103285Sikobstatic SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 158106804SsimokawaSYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 159103285Sikob "Total bounce pages"); 160103285Sikob 161103285Sikob#define DMAMAP_LINEAR 0x1 162103285Sikob#define DMAMAP_MBUF 0x2 163103285Sikob#define DMAMAP_UIO 0x4 164106810Ssimokawa#define DMAMAP_CACHE_ALIGNED 0x10 165106810Ssimokawa#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) 166106810Ssimokawa#define DMAMAP_COHERENT 0x8 167129585Sdfrstruct bus_dmamap { 168129585Sdfr struct bp_list bpages; 169129585Sdfr int pagesneeded; 170120660Ssimokawa int pagesreserved; 171106810Ssimokawa bus_dma_tag_t dmat; 172106810Ssimokawa int flags; 173106810Ssimokawa void *buffer; 174106810Ssimokawa int len; 175106810Ssimokawa STAILQ_ENTRY(bus_dmamap) links; 176120660Ssimokawa bus_dmamap_callback_t *callback; 177106810Ssimokawa void *callback_arg; 178106810Ssimokawa 179106810Ssimokawa}; 180120660Ssimokawa 181110072Ssimokawastatic STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 182113584Ssimokawastatic STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 183113584Ssimokawa 184129585Sdfrstatic struct mtx busdma_mtx; 185106810Ssimokawa 186120660SsimokawaMTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); 187106810Ssimokawa 188106810Ssimokawastatic void init_bounce_pages(void *dummy); 189106816Ssimokawastatic int alloc_bounce_zone(bus_dma_tag_t dmat); 190129585Sdfrstatic int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 191106810Ssimokawastatic int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 192106810Ssimokawa int commit); 193106810Ssimokawastatic bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 194106810Ssimokawa vm_offset_t vaddr, bus_size_t size); 195106810Ssimokawastatic void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 196106810Ssimokawa 197106810Ssimokawa/* Default tag, as most drivers provide no parent tag. */ 198106810Ssimokawabus_dma_tag_t arm_root_dma_tag; 199106810Ssimokawa 200103285Sikob/* 201106810Ssimokawa * ---------------------------------------------------------------------------- 202106810Ssimokawa * Begin block of code useful to transplant to other implementations. 203129585Sdfr */ 204129585Sdfr 205129585Sdfrstatic uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ 206106804Ssimokawa 207120660Ssimokawastatic busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ 208106804Ssimokawastatic busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ 209103285Sikob 210103285Sikob/* 211103285Sikob * This is the ctor function passed to uma_zcreate() for the pool of dma maps. 212120660Ssimokawa * It'll need platform-specific changes if this code is copied. 213120660Ssimokawa */ 214106804Ssimokawastatic int 215103285Sikobdmamap_ctor(void *mem, int size, void *arg, int flags) 216106804Ssimokawa{ 217120660Ssimokawa bus_dmamap_t map; 218103285Sikob bus_dma_tag_t dmat; 219113584Ssimokawa 220113584Ssimokawa map = (bus_dmamap_t)mem; 221113584Ssimokawa dmat = (bus_dma_tag_t)arg; 222120660Ssimokawa 223103285Sikob dmat->map_count++; 224120660Ssimokawa 225120660Ssimokawa map->dmat = dmat; 226120660Ssimokawa map->flags = 0; 227103285Sikob STAILQ_INIT(&map->bpages); 228106816Ssimokawa 229106810Ssimokawa return (0); 230106810Ssimokawa} 231103285Sikob 232106804Ssimokawa/* 233103285Sikob * This is the dtor function passed to uma_zcreate() for the pool of dma maps. 234103285Sikob * It may need platform-specific changes if this code is copied . 235103285Sikob */ 236103285Sikobstatic void 237110337Ssimokawadmamap_dtor(void *mem, int size, void *arg) 238110337Ssimokawa{ 239110337Ssimokawa bus_dmamap_t map; 240110337Ssimokawa 241129585Sdfr map = (bus_dmamap_t)mem; 242129585Sdfr 243129585Sdfr map->dmat->map_count--; 244110337Ssimokawa} 245120660Ssimokawa 246110337Ssimokawastatic void 247110337Ssimokawabusdma_init(void *dummy) 248110337Ssimokawa{ 249110337Ssimokawa 250110337Ssimokawa /* Create a cache of maps for bus_dmamap_create(). */ 251120660Ssimokawa dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), 252110337Ssimokawa dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 253110337Ssimokawa 254110337Ssimokawa /* Create a cache of buffers in standard (cacheable) memory. */ 255120660Ssimokawa standard_allocator = busdma_bufalloc_create("buffer", 256110406Ssimokawa arm_dcache_align, /* minimum_alignment */ 257113584Ssimokawa NULL, /* uma_alloc func */ 258113584Ssimokawa NULL, /* uma_free func */ 259113584Ssimokawa 0); /* uma_zcreate_flags */ 260120660Ssimokawa 261110337Ssimokawa /* 262120660Ssimokawa * Create a cache of buffers in uncacheable memory, to implement the 263120660Ssimokawa * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. 264120660Ssimokawa */ 265110337Ssimokawa coherent_allocator = busdma_bufalloc_create("coherent", 266110337Ssimokawa arm_dcache_align, /* minimum_alignment */ 267110337Ssimokawa busdma_bufalloc_alloc_uncacheable, 268110337Ssimokawa busdma_bufalloc_free_uncacheable, 269110337Ssimokawa 0); /* uma_zcreate_flags */ 270110337Ssimokawa} 271110337Ssimokawa 272110337Ssimokawa/* 273110337Ssimokawa * This init historically used SI_SUB_VM, but now the init code requires 274110337Ssimokawa * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by 275110337Ssimokawa * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using 276103285Sikob * SI_SUB_KMEM and SI_ORDER_THIRD. 277130585Sphk */ 278103285SikobSYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); 279122228Ssimokawa 280110582Ssimokawa/* 281122228Ssimokawa * End block of code useful to transplant to other implementations. 282122228Ssimokawa * ---------------------------------------------------------------------------- 283122228Ssimokawa */ 284122228Ssimokawa 285122228Ssimokawa/* 286122228Ssimokawa * Return true if a match is made. 287122228Ssimokawa * 288124145Ssimokawa * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 289122228Ssimokawa * 290122228Ssimokawa * If paddr is within the bounds of the dma tag then call the filter callback 291122228Ssimokawa * to check for a match, if there is no filter callback then assume a match. 292122228Ssimokawa */ 293122228Ssimokawastatic int 294122228Ssimokawarun_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 295122228Ssimokawa{ 296122228Ssimokawa int retval; 297127468Ssimokawa 298110582Ssimokawa retval = 0; 299110582Ssimokawa 300103285Sikob do { 301103285Sikob if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 302103285Sikob || ((paddr & (dmat->alignment - 1)) != 0)) 303130585Sphk && (dmat->filter == NULL 304103285Sikob || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 305122228Ssimokawa retval = 1; 306115786Ssimokawa 307122228Ssimokawa dmat = dmat->parent; 308122228Ssimokawa } while (retval == 0 && dmat != NULL); 309122228Ssimokawa return (retval); 310127468Ssimokawa} 311122228Ssimokawa 312137503Ssimokawa/* 313122228Ssimokawa * This routine checks the exclusion zone constraints from a tag against the 314122228Ssimokawa * physical RAM available on the machine. If a tag specifies an exclusion zone 315122228Ssimokawa * but there's no RAM in that zone, then we avoid allocating resources to bounce 316110582Ssimokawa * a request, and we can use any memory allocator (as opposed to needing 317103285Sikob * kmem_alloc_contig() just because it can allocate pages in an address range). 318103285Sikob * 319120660Ssimokawa * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the 320120660Ssimokawa * same value on 32-bit architectures) as their lowaddr constraint, and we can't 321120660Ssimokawa * possibly have RAM at an address higher than the highest address we can 322103285Sikob * express, so we take a fast out. 323120660Ssimokawa */ 324103285Sikobstatic __inline int 325120660Ssimokawa_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) 326120660Ssimokawa{ 327120660Ssimokawa int i; 328120660Ssimokawa 329121381Ssimokawa if (lowaddr >= BUS_SPACE_MAXADDR) 330127468Ssimokawa return (0); 331120660Ssimokawa 332120660Ssimokawa for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { 333106810Ssimokawa if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) 334103285Sikob || (lowaddr < phys_avail[i] && 335120660Ssimokawa highaddr > phys_avail[i])) 336120660Ssimokawa return (1); 337103285Sikob } 338120660Ssimokawa return (0); 339120660Ssimokawa} 340120660Ssimokawa 341103285Sikobstatic __inline struct arm32_dma_range * 342110337Ssimokawa_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 343122228Ssimokawa bus_addr_t curaddr) 344110337Ssimokawa{ 345110337Ssimokawa struct arm32_dma_range *dr; 346130585Sphk int i; 347120660Ssimokawa 348110337Ssimokawa for (i = 0, dr = ranges; i < nranges; i++, dr++) { 349120660Ssimokawa if (curaddr >= dr->dr_sysbase && 350120660Ssimokawa round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 351120660Ssimokawa return (dr); 352120660Ssimokawa } 353110337Ssimokawa 354120660Ssimokawa return (NULL); 355120660Ssimokawa} 356122228Ssimokawa/* 357122228Ssimokawa * Convenience function for manipulating driver locks from busdma (during 358110337Ssimokawa * busdma_swi, for example). Drivers that don't provide their own locks 359110577Ssimokawa * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 360110577Ssimokawa * non-mutex locking scheme don't have to use this at all. 361122228Ssimokawa */ 362120660Ssimokawavoid 363120660Ssimokawabusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 364110337Ssimokawa{ 365110337Ssimokawa struct mtx *dmtx; 366120660Ssimokawa 367120660Ssimokawa dmtx = (struct mtx *)arg; 368120660Ssimokawa switch (op) { 369120660Ssimokawa case BUS_DMA_LOCK: 370120660Ssimokawa mtx_lock(dmtx); 371120660Ssimokawa break; 372120660Ssimokawa case BUS_DMA_UNLOCK: 373120660Ssimokawa mtx_unlock(dmtx); 374120660Ssimokawa break; 375120660Ssimokawa default: 376120660Ssimokawa panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 377120660Ssimokawa } 378120660Ssimokawa} 379120660Ssimokawa 380120660Ssimokawa/* 381120660Ssimokawa * dflt_lock should never get called. It gets put into the dma tag when 382120660Ssimokawa * lockfunc == NULL, which is only valid if the maps that are associated 383120660Ssimokawa * with the tag are meant to never be defered. 384120660Ssimokawa * XXX Should have a way to identify which driver is responsible here. 385120660Ssimokawa */ 386120660Ssimokawastatic void 387120660Ssimokawadflt_lock(void *arg, bus_dma_lock_op_t op) 388120660Ssimokawa{ 389110337Ssimokawa#ifdef INVARIANTS 390120660Ssimokawa panic("driver error: busdma dflt_lock called"); 391120660Ssimokawa#else 392120660Ssimokawa printf("DRIVER_ERROR: busdma dflt_lock called\n"); 393120660Ssimokawa#endif 394120660Ssimokawa} 395120660Ssimokawa 396120660Ssimokawa/* 397120660Ssimokawa * Allocate a device specific dma_tag. 398120660Ssimokawa */ 399121381Ssimokawa#define SEG_NB 1024 400127468Ssimokawa 401120660Ssimokawaint 402120660Ssimokawabus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 403120660Ssimokawa bus_addr_t boundary, bus_addr_t lowaddr, 404120660Ssimokawa bus_addr_t highaddr, bus_dma_filter_t *filter, 405120660Ssimokawa void *filterarg, bus_size_t maxsize, int nsegments, 406103285Sikob bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 407110337Ssimokawa void *lockfuncarg, bus_dma_tag_t *dmat) 408103285Sikob{ 409130585Sphk bus_dma_tag_t newtag; 410103285Sikob int error = 0; 411122228Ssimokawa /* Return a NULL tag on failure */ 412110582Ssimokawa *dmat = NULL; 413122228Ssimokawa if (!parent) 414122228Ssimokawa parent = arm_root_dma_tag; 415110582Ssimokawa 416110582Ssimokawa newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 417122228Ssimokawa if (newtag == NULL) { 418110582Ssimokawa CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 419110582Ssimokawa __func__, newtag, 0, error); 420122228Ssimokawa return (ENOMEM); 421110582Ssimokawa } 422110582Ssimokawa 423110582Ssimokawa newtag->parent = parent; 424110582Ssimokawa newtag->alignment = alignment ? alignment : 1; 425110582Ssimokawa newtag->boundary = boundary; 426103285Sikob newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 427103285Sikob newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 428130585Sphk newtag->filter = filter; 429103285Sikob newtag->filterarg = filterarg; 430103285Sikob newtag->maxsize = maxsize; 431103285Sikob newtag->nsegments = nsegments; 432103285Sikob newtag->maxsegsz = maxsegsz; 433127468Ssimokawa newtag->flags = flags; 434130585Sphk newtag->ref_count = 1; /* Count ourself */ 435111615Ssimokawa newtag->map_count = 0; 436130585Sphk newtag->ranges = bus_dma_get_range(); 437111615Ssimokawa newtag->_nranges = bus_dma_get_range_nb(); 438103285Sikob if (lockfunc != NULL) { 439103285Sikob newtag->lockfunc = lockfunc; 440103285Sikob newtag->lockfuncarg = lockfuncarg; 441 } else { 442 newtag->lockfunc = dflt_lock; 443 newtag->lockfuncarg = NULL; 444 } 445 /* 446 * If all the segments we need fit into the local tagsegs array, set the 447 * pointer now. Otherwise NULL the pointer and an array of segments 448 * will be allocated later, on first use. We don't pre-allocate now 449 * because some tags exist just to pass contraints to children in the 450 * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we 451 * sure don't want to try to allocate an array for that. 452 */ 453 if (newtag->nsegments <= nitems(newtag->tagsegs)) 454 newtag->segments = newtag->tagsegs; 455 else 456 newtag->segments = NULL; 457 /* 458 * Take into account any restrictions imposed by our parent tag 459 */ 460 if (parent != NULL) { 461 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 462 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 463 if (newtag->boundary == 0) 464 newtag->boundary = parent->boundary; 465 else if (parent->boundary != 0) 466 newtag->boundary = MIN(parent->boundary, 467 newtag->boundary); 468 if ((newtag->filter != NULL) || 469 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 470 newtag->flags |= BUS_DMA_COULD_BOUNCE; 471 if (newtag->filter == NULL) { 472 /* 473 * Short circuit looking at our parent directly 474 * since we have encapsulated all of its information 475 */ 476 newtag->filter = parent->filter; 477 newtag->filterarg = parent->filterarg; 478 newtag->parent = parent->parent; 479 } 480 if (newtag->parent != NULL) 481 atomic_add_int(&parent->ref_count, 1); 482 } 483 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr) 484 || newtag->alignment > 1) 485 newtag->flags |= BUS_DMA_COULD_BOUNCE; 486 487 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 488 (flags & BUS_DMA_ALLOCNOW) != 0) { 489 struct bounce_zone *bz; 490 491 /* Must bounce */ 492 493 if ((error = alloc_bounce_zone(newtag)) != 0) { 494 free(newtag, M_DEVBUF); 495 return (error); 496 } 497 bz = newtag->bounce_zone; 498 499 if (ptoa(bz->total_bpages) < maxsize) { 500 int pages; 501 502 pages = atop(maxsize) - bz->total_bpages; 503 504 /* Add pages to our bounce pool */ 505 if (alloc_bounce_pages(newtag, pages) < pages) 506 error = ENOMEM; 507 } 508 /* Performed initial allocation */ 509 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 510 } else 511 newtag->bounce_zone = NULL; 512 if (error != 0) 513 free(newtag, M_DEVBUF); 514 else 515 *dmat = newtag; 516 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 517 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 518 519 return (error); 520} 521 522int 523bus_dma_tag_destroy(bus_dma_tag_t dmat) 524{ 525#ifdef KTR 526 bus_dma_tag_t dmat_copy = dmat; 527#endif 528 529 if (dmat != NULL) { 530 531 if (dmat->map_count != 0) 532 return (EBUSY); 533 534 while (dmat != NULL) { 535 bus_dma_tag_t parent; 536 537 parent = dmat->parent; 538 atomic_subtract_int(&dmat->ref_count, 1); 539 if (dmat->ref_count == 0) { 540 if (dmat->segments != NULL && 541 dmat->segments != dmat->tagsegs) 542 free(dmat->segments, M_DEVBUF); 543 free(dmat, M_DEVBUF); 544 /* 545 * Last reference count, so 546 * release our reference 547 * count on our parent. 548 */ 549 dmat = parent; 550 } else 551 dmat = NULL; 552 } 553 } 554 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 555 556 return (0); 557} 558 559#include <sys/kdb.h> 560/* 561 * Allocate a handle for mapping from kva/uva/physical 562 * address space into bus device space. 563 */ 564int 565bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 566{ 567 bus_dmamap_t map; 568 int error = 0; 569 570 map = uma_zalloc_arg(dmamap_zone, dmat, M_WAITOK); 571 *mapp = map; 572 573 /* 574 * If the tag's segments haven't been allocated yet we need to do it 575 * now, because we can't sleep for resources at map load time. 576 */ 577 if (dmat->segments == NULL) 578 dmat->segments = malloc(dmat->nsegments * 579 sizeof(*dmat->segments), M_DEVBUF, M_WAITOK); 580 581 /* 582 * Bouncing might be required if the driver asks for an active 583 * exclusion region, a data alignment that is stricter than 1, and/or 584 * an active address boundary. 585 */ 586 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 587 588 /* Must bounce */ 589 struct bounce_zone *bz; 590 int maxpages; 591 592 if (dmat->bounce_zone == NULL) { 593 if ((error = alloc_bounce_zone(dmat)) != 0) { 594 uma_zfree(dmamap_zone, map); 595 *mapp = NULL; 596 return (error); 597 } 598 } 599 bz = dmat->bounce_zone; 600 601 /* Initialize the new map */ 602 STAILQ_INIT(&((*mapp)->bpages)); 603 604 /* 605 * Attempt to add pages to our pool on a per-instance 606 * basis up to a sane limit. 607 */ 608 maxpages = MAX_BPAGES; 609 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 610 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 611 int pages; 612 613 pages = MAX(atop(dmat->maxsize), 1); 614 pages = MIN(maxpages - bz->total_bpages, pages); 615 pages = MAX(pages, 1); 616 if (alloc_bounce_pages(dmat, pages) < pages) 617 error = ENOMEM; 618 619 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 620 if (error == 0) 621 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 622 } else { 623 error = 0; 624 } 625 } 626 bz->map_count++; 627 } 628 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 629 __func__, dmat, dmat->flags, error); 630 631 return (0); 632} 633 634/* 635 * Destroy a handle for mapping from kva/uva/physical 636 * address space into bus device space. 637 */ 638int 639bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 640{ 641 642 if (STAILQ_FIRST(&map->bpages) != NULL) { 643 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 644 __func__, dmat, EBUSY); 645 return (EBUSY); 646 } 647 uma_zfree(dmamap_zone, map); 648 if (dmat->bounce_zone) 649 dmat->bounce_zone->map_count--; 650 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 651 return (0); 652} 653 654/* 655 * Allocate a piece of memory that can be efficiently mapped into bus device 656 * space based on the constraints listed in the dma tag. Returns a pointer to 657 * the allocated memory, and a pointer to an associated bus_dmamap. 658 */ 659int 660bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, 661 bus_dmamap_t *mapp) 662{ 663 void * vaddr; 664 struct busdma_bufzone *bufzone; 665 busdma_bufalloc_t ba; 666 bus_dmamap_t map; 667 int mflags; 668 vm_memattr_t memattr; 669 670 if (flags & BUS_DMA_NOWAIT) 671 mflags = M_NOWAIT; 672 else 673 mflags = M_WAITOK; 674 675 /* 676 * If the tag's segments haven't been allocated yet we need to do it 677 * now, because we can't sleep for resources at map load time. 678 */ 679 if (dmat->segments == NULL) 680 dmat->segments = malloc(dmat->nsegments * 681 sizeof(*dmat->segments), M_DEVBUF, mflags); 682 683 map = uma_zalloc_arg(dmamap_zone, dmat, mflags); 684 if (map == NULL) 685 return (ENOMEM); 686 687 if (flags & BUS_DMA_COHERENT) { 688 memattr = VM_MEMATTR_UNCACHEABLE; 689 ba = coherent_allocator; 690 map->flags |= DMAMAP_COHERENT; 691 } else { 692 memattr = VM_MEMATTR_DEFAULT; 693 ba = standard_allocator; 694 } 695 /* All buffers we allocate are cache-aligned. */ 696 map->flags |= DMAMAP_CACHE_ALIGNED; 697 698 if (flags & BUS_DMA_ZERO) 699 mflags |= M_ZERO; 700 701 /* 702 * Try to find a bufzone in the allocator that holds a cache of buffers 703 * of the right size for this request. If the buffer is too big to be 704 * held in the allocator cache, this returns NULL. 705 */ 706 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 707 708 /* 709 * Allocate the buffer from the uma(9) allocator if... 710 * - It's small enough to be in the allocator (bufzone not NULL). 711 * - The alignment constraint isn't larger than the allocation size 712 * (the allocator aligns buffers to their size boundaries). 713 * - There's no need to handle lowaddr/highaddr exclusion zones. 714 * else allocate non-contiguous pages if... 715 * - The page count that could get allocated doesn't exceed nsegments. 716 * - The alignment constraint isn't larger than a page boundary. 717 * - There are no boundary-crossing constraints. 718 * else allocate a block of contiguous pages because one or more of the 719 * constraints is something that only the contig allocator can fulfill. 720 */ 721 if (bufzone != NULL && dmat->alignment <= bufzone->size && 722 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { 723 vaddr = uma_zalloc(bufzone->umazone, mflags); 724 } else if (dmat->nsegments >= btoc(dmat->maxsize) && 725 dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { 726 vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize, 727 mflags, 0, dmat->lowaddr, memattr); 728 } else { 729 vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize, 730 mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, 731 memattr); 732 } 733 734 if (vaddr == NULL) { 735 uma_zfree(dmamap_zone, map); 736 map = NULL; 737 } 738 739 *vaddrp = vaddr; 740 *mapp = map; 741 742 return (vaddr == NULL ? ENOMEM : 0); 743} 744 745/* 746 * Free a piece of memory that was allocated via bus_dmamem_alloc, along with 747 * its associated map. 748 */ 749void 750bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 751{ 752 struct busdma_bufzone *bufzone; 753 busdma_bufalloc_t ba; 754 755 if (map->flags & DMAMAP_COHERENT) 756 ba = coherent_allocator; 757 else 758 ba = standard_allocator; 759 uma_zfree(dmamap_zone, map); 760 761 /* Be careful not to access map from here on. */ 762 763 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); 764 765 if (bufzone != NULL && dmat->alignment <= bufzone->size && 766 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) 767 uma_zfree(bufzone->umazone, vaddr); 768 else 769 kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); 770} 771 772static int 773_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 774 void *buf, bus_size_t buflen, int flags) 775{ 776 vm_offset_t vaddr; 777 vm_offset_t vendaddr; 778 bus_addr_t paddr; 779 780 if ((map->pagesneeded == 0)) { 781 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", 782 dmat->lowaddr, dmat->boundary, dmat->alignment); 783 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", 784 map, map->pagesneeded); 785 /* 786 * Count the number of bounce pages 787 * needed in order to complete this transfer 788 */ 789 vaddr = trunc_page((vm_offset_t)buf); 790 vendaddr = (vm_offset_t)buf + buflen; 791 792 while (vaddr < vendaddr) { 793 if (__predict_true(pmap == pmap_kernel())) 794 paddr = pmap_kextract(vaddr); 795 else 796 paddr = pmap_extract(pmap, vaddr); 797 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 798 run_filter(dmat, paddr) != 0) 799 map->pagesneeded++; 800 vaddr += PAGE_SIZE; 801 } 802 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 803 } 804 805 /* Reserve Necessary Bounce Pages */ 806 if (map->pagesneeded != 0) { 807 mtx_lock(&bounce_lock); 808 if (flags & BUS_DMA_NOWAIT) { 809 if (reserve_bounce_pages(dmat, map, 0) != 0) { 810 mtx_unlock(&bounce_lock); 811 return (ENOMEM); 812 } 813 } else { 814 if (reserve_bounce_pages(dmat, map, 1) != 0) { 815 /* Queue us for resources */ 816 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 817 map, links); 818 mtx_unlock(&bounce_lock); 819 return (EINPROGRESS); 820 } 821 } 822 mtx_unlock(&bounce_lock); 823 } 824 825 return (0); 826} 827 828/* 829 * Utility function to load a linear buffer. lastaddrp holds state 830 * between invocations (for multiple-buffer loads). segp contains 831 * the starting segment on entrance, and the ending segment on exit. 832 * first indicates if this is the first invocation of this function. 833 */ 834static __inline int 835bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 836 bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 837 int flags, vm_offset_t *lastaddrp, int *segp) 838{ 839 bus_size_t sgsize; 840 bus_addr_t curaddr, lastaddr, baddr, bmask; 841 vm_offset_t vaddr = (vm_offset_t)buf; 842 int seg; 843 int error = 0; 844 pd_entry_t *pde; 845 pt_entry_t pte; 846 pt_entry_t *ptep; 847 848 lastaddr = *lastaddrp; 849 bmask = ~(dmat->boundary - 1); 850 851 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 852 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, 853 flags); 854 if (error) 855 return (error); 856 } 857 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 858 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 859 860 for (seg = *segp; buflen > 0 ; ) { 861 /* 862 * Get the physical address for this segment. 863 * 864 * XXX Don't support checking for coherent mappings 865 * XXX in user address space. 866 */ 867 if (__predict_true(pmap == pmap_kernel())) { 868 if (pmap_get_pde_pte(pmap, vaddr, &pde, &ptep) == FALSE) 869 return (EFAULT); 870 871 if (__predict_false(pmap_pde_section(pde))) { 872 if (*pde & L1_S_SUPERSEC) 873 curaddr = (*pde & L1_SUP_FRAME) | 874 (vaddr & L1_SUP_OFFSET); 875 else 876 curaddr = (*pde & L1_S_FRAME) | 877 (vaddr & L1_S_OFFSET); 878 } else { 879 pte = *ptep; 880 KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV, 881 ("INV type")); 882 if (__predict_false((pte & L2_TYPE_MASK) 883 == L2_TYPE_L)) { 884 curaddr = (pte & L2_L_FRAME) | 885 (vaddr & L2_L_OFFSET); 886 } else { 887 curaddr = (pte & L2_S_FRAME) | 888 (vaddr & L2_S_OFFSET); 889 } 890 } 891 } else { 892 curaddr = pmap_extract(pmap, vaddr); 893 map->flags &= ~DMAMAP_COHERENT; 894 } 895 896 /* 897 * Compute the segment size, and adjust counts. 898 */ 899 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 900 if (sgsize > dmat->maxsegsz) 901 sgsize = dmat->maxsegsz; 902 if (buflen < sgsize) 903 sgsize = buflen; 904 905 /* 906 * Make sure we don't cross any boundaries. 907 */ 908 if (dmat->boundary > 0) { 909 baddr = (curaddr + dmat->boundary) & bmask; 910 if (sgsize > (baddr - curaddr)) 911 sgsize = (baddr - curaddr); 912 } 913 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 914 map->pagesneeded != 0 && run_filter(dmat, curaddr)) 915 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 916 917 if (dmat->ranges) { 918 struct arm32_dma_range *dr; 919 920 dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 921 curaddr); 922 if (dr == NULL) 923 return (EINVAL); 924 /* 925 * In a valid DMA range. Translate the physical 926 * memory address to an address in the DMA window. 927 */ 928 curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 929 930 } 931 932 /* 933 * Insert chunk into a segment, coalescing with 934 * the previous segment if possible. 935 */ 936 if (seg >= 0 && curaddr == lastaddr && 937 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 938 (dmat->boundary == 0 || 939 (segs[seg].ds_addr & bmask) == 940 (curaddr & bmask))) { 941 segs[seg].ds_len += sgsize; 942 goto segdone; 943 } else { 944 if (++seg >= dmat->nsegments) 945 break; 946 segs[seg].ds_addr = curaddr; 947 segs[seg].ds_len = sgsize; 948 } 949 if (error) 950 break; 951segdone: 952 lastaddr = curaddr + sgsize; 953 vaddr += sgsize; 954 buflen -= sgsize; 955 } 956 957 *segp = seg; 958 *lastaddrp = lastaddr; 959 960 /* 961 * Did we fit? 962 */ 963 if (buflen != 0) 964 error = EFBIG; /* XXX better return value here? */ 965 return (error); 966} 967 968/* 969 * Map the buffer buf into bus space using the dmamap map. 970 */ 971int 972bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 973 bus_size_t buflen, bus_dmamap_callback_t *callback, 974 void *callback_arg, int flags) 975{ 976 vm_offset_t lastaddr = 0; 977 int error, nsegs = -1; 978 979 KASSERT(dmat != NULL, ("dmatag is NULL")); 980 KASSERT(map != NULL, ("dmamap is NULL")); 981 map->callback = callback; 982 map->callback_arg = callback_arg; 983 map->flags &= ~DMAMAP_TYPE_MASK; 984 map->flags |= DMAMAP_LINEAR; 985 map->buffer = buf; 986 map->len = buflen; 987 error = bus_dmamap_load_buffer(dmat, 988 dmat->segments, map, buf, buflen, kernel_pmap, 989 flags, &lastaddr, &nsegs); 990 if (error == EINPROGRESS) 991 return (error); 992 if (error) 993 (*callback)(callback_arg, NULL, 0, error); 994 else 995 (*callback)(callback_arg, dmat->segments, nsegs + 1, error); 996 997 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 998 __func__, dmat, dmat->flags, nsegs + 1, error); 999 1000 return (error); 1001} 1002 1003/* 1004 * Like bus_dmamap_load(), but for mbufs. 1005 * 1006 * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs. 1007 * 1008 * We know that the way the system allocates and uses mbufs implies that we can 1009 * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line 1010 * flushes. Even though the flush may reference the data area within the mbuf 1011 * that isn't aligned to a cache line, we know the overall mbuf itself is 1012 * properly aligned, and we know that the CPU will not touch the header fields 1013 * before the data area while the DMA is in progress. 1014 */ 1015int 1016bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 1017 bus_dmamap_callback2_t *callback, void *callback_arg, 1018 int flags) 1019{ 1020 int nsegs = -1, error = 0; 1021 1022 M_ASSERTPKTHDR(m0); 1023 1024 flags |= BUS_DMA_NOWAIT; 1025 map->flags &= ~DMAMAP_TYPE_MASK; 1026 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1027 map->buffer = m0; 1028 map->len = 0; 1029 if (m0->m_pkthdr.len <= dmat->maxsize) { 1030 vm_offset_t lastaddr = 0; 1031 struct mbuf *m; 1032 1033 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1034 if (m->m_len > 0) { 1035 error = bus_dmamap_load_buffer(dmat, 1036 dmat->segments, map, m->m_data, m->m_len, 1037 pmap_kernel(), flags, &lastaddr, &nsegs); 1038 map->len += m->m_len; 1039 } 1040 } 1041 } else { 1042 error = EINVAL; 1043 } 1044 1045 if (error) { 1046 /* 1047 * force "no valid mappings" on error in callback. 1048 */ 1049 (*callback)(callback_arg, NULL, 0, 0, error); 1050 } else { 1051 (*callback)(callback_arg, dmat->segments, nsegs + 1, 1052 m0->m_pkthdr.len, error); 1053 } 1054 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1055 __func__, dmat, dmat->flags, error, nsegs + 1); 1056 1057 return (error); 1058} 1059 1060int 1061bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 1062 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 1063 int flags) 1064{ 1065 int error = 0; 1066 M_ASSERTPKTHDR(m0); 1067 1068 flags |= BUS_DMA_NOWAIT; 1069 *nsegs = -1; 1070 map->flags &= ~DMAMAP_TYPE_MASK; 1071 map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; 1072 map->buffer = m0; 1073 map->len = 0; 1074 if (m0->m_pkthdr.len <= dmat->maxsize) { 1075 vm_offset_t lastaddr = 0; 1076 struct mbuf *m; 1077 1078 for (m = m0; m != NULL && error == 0; m = m->m_next) { 1079 if (m->m_len > 0) { 1080 error = bus_dmamap_load_buffer(dmat, segs, map, 1081 m->m_data, m->m_len, 1082 pmap_kernel(), flags, &lastaddr, 1083 nsegs); 1084 map->len += m->m_len; 1085 } 1086 } 1087 } else { 1088 error = EINVAL; 1089 } 1090 1091 /* XXX FIXME: Having to increment nsegs is really annoying */ 1092 ++*nsegs; 1093 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1094 __func__, dmat, dmat->flags, error, *nsegs); 1095 return (error); 1096} 1097 1098/* 1099 * Like bus_dmamap_load(), but for uios. 1100 */ 1101int 1102bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 1103 bus_dmamap_callback2_t *callback, void *callback_arg, 1104 int flags) 1105{ 1106 vm_offset_t lastaddr = 0; 1107 int nsegs, i, error; 1108 bus_size_t resid; 1109 struct iovec *iov; 1110 struct pmap *pmap; 1111 1112 resid = uio->uio_resid; 1113 iov = uio->uio_iov; 1114 map->flags &= ~DMAMAP_TYPE_MASK; 1115 map->flags |= DMAMAP_UIO; 1116 map->buffer = uio; 1117 map->len = 0; 1118 1119 if (uio->uio_segflg == UIO_USERSPACE) { 1120 KASSERT(uio->uio_td != NULL, 1121 ("bus_dmamap_load_uio: USERSPACE but no proc")); 1122 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 1123 } else 1124 pmap = kernel_pmap; 1125 1126 error = 0; 1127 nsegs = -1; 1128 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 1129 /* 1130 * Now at the first iovec to load. Load each iovec 1131 * until we have exhausted the residual count. 1132 */ 1133 bus_size_t minlen = 1134 resid < iov[i].iov_len ? resid : iov[i].iov_len; 1135 caddr_t addr = (caddr_t) iov[i].iov_base; 1136 1137 if (minlen > 0) { 1138 error = bus_dmamap_load_buffer(dmat, dmat->segments, 1139 map, addr, minlen, pmap, flags, &lastaddr, &nsegs); 1140 1141 map->len += minlen; 1142 resid -= minlen; 1143 } 1144 } 1145 1146 if (error) { 1147 /* 1148 * force "no valid mappings" on error in callback. 1149 */ 1150 (*callback)(callback_arg, NULL, 0, 0, error); 1151 } else { 1152 (*callback)(callback_arg, dmat->segments, nsegs+1, 1153 uio->uio_resid, error); 1154 } 1155 1156 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 1157 __func__, dmat, dmat->flags, error, nsegs + 1); 1158 return (error); 1159} 1160 1161/* 1162 * Release the mapping held by map. 1163 */ 1164void 1165_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1166{ 1167 struct bounce_page *bpage; 1168 1169 map->flags &= ~DMAMAP_TYPE_MASK; 1170 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1171 STAILQ_REMOVE_HEAD(&map->bpages, links); 1172 free_bounce_page(dmat, bpage); 1173 } 1174 return; 1175} 1176 1177static void 1178bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned) 1179{ 1180 char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; 1181 register_t s; 1182 int partial; 1183 1184 if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { 1185 cpu_dcache_wb_range((vm_offset_t)buf, len); 1186 cpu_l2cache_wb_range((vm_offset_t)buf, len); 1187 } 1188 1189 /* 1190 * If the caller promises the buffer is properly aligned to a cache line 1191 * (even if the call parms make it look like it isn't) we can avoid 1192 * attempting to preserve the non-DMA part of the cache line in the 1193 * POSTREAD case, but we MUST still do a writeback in the PREREAD case. 1194 * 1195 * This covers the case of mbufs, where we know how they're aligned and 1196 * know the CPU doesn't touch the header in front of the DMA data area 1197 * during the IO, but it may have touched it right before invoking the 1198 * sync, so a PREREAD writeback is required. 1199 * 1200 * It also handles buffers we created in bus_dmamem_alloc(), which are 1201 * always aligned and padded to cache line size even if the IO length 1202 * isn't a multiple of cache line size. In this case the PREREAD 1203 * writeback probably isn't required, but it's harmless. 1204 */ 1205 partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; 1206 1207 if (op & BUS_DMASYNC_PREREAD) { 1208 if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { 1209 cpu_dcache_inv_range((vm_offset_t)buf, len); 1210 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1211 } else { 1212 cpu_dcache_wbinv_range((vm_offset_t)buf, len); 1213 cpu_l2cache_wbinv_range((vm_offset_t)buf, len); 1214 } 1215 } 1216 if (op & BUS_DMASYNC_POSTREAD) { 1217 if (partial && !bufaligned) { 1218 s = intr_disable(); 1219 if ((vm_offset_t)buf & arm_dcache_align_mask) 1220 memcpy(_tmp_cl, (void *)((vm_offset_t)buf & 1221 ~arm_dcache_align_mask), 1222 (vm_offset_t)buf & arm_dcache_align_mask); 1223 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1224 memcpy(_tmp_clend, 1225 (void *)((vm_offset_t)buf + len), 1226 arm_dcache_align - (((vm_offset_t)(buf) + 1227 len) & arm_dcache_align_mask)); 1228 } 1229 cpu_dcache_inv_range((vm_offset_t)buf, len); 1230 cpu_l2cache_inv_range((vm_offset_t)buf, len); 1231 if (partial && !bufaligned) { 1232 if ((vm_offset_t)buf & arm_dcache_align_mask) 1233 memcpy((void *)((vm_offset_t)buf & 1234 ~arm_dcache_align_mask), _tmp_cl, 1235 (vm_offset_t)buf & arm_dcache_align_mask); 1236 if (((vm_offset_t)buf + len) & arm_dcache_align_mask) 1237 memcpy((void *)((vm_offset_t)buf + len), 1238 _tmp_clend, arm_dcache_align - 1239 (((vm_offset_t)(buf) + len) & 1240 arm_dcache_align_mask)); 1241 intr_restore(s); 1242 } 1243 } 1244} 1245 1246static void 1247_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1248{ 1249 struct bounce_page *bpage; 1250 1251 STAILQ_FOREACH(bpage, &map->bpages, links) { 1252 if (op & BUS_DMASYNC_PREWRITE) { 1253 bcopy((void *)bpage->datavaddr, 1254 (void *)(bpage->vaddr_nocache != 0 ? 1255 bpage->vaddr_nocache : bpage->vaddr), 1256 bpage->datacount); 1257 if (bpage->vaddr_nocache == 0) { 1258 cpu_dcache_wb_range(bpage->vaddr, 1259 bpage->datacount); 1260 cpu_l2cache_wb_range(bpage->vaddr, 1261 bpage->datacount); 1262 } 1263 dmat->bounce_zone->total_bounced++; 1264 } 1265 if (op & BUS_DMASYNC_POSTREAD) { 1266 if (bpage->vaddr_nocache == 0) { 1267 cpu_dcache_inv_range(bpage->vaddr, 1268 bpage->datacount); 1269 cpu_l2cache_inv_range(bpage->vaddr, 1270 bpage->datacount); 1271 } 1272 bcopy((void *)(bpage->vaddr_nocache != 0 ? 1273 bpage->vaddr_nocache : bpage->vaddr), 1274 (void *)bpage->datavaddr, bpage->datacount); 1275 dmat->bounce_zone->total_bounced++; 1276 } 1277 } 1278} 1279 1280static __inline int 1281_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len) 1282{ 1283 struct bounce_page *bpage; 1284 1285 STAILQ_FOREACH(bpage, &map->bpages, links) { 1286 if ((vm_offset_t)buf >= bpage->datavaddr && 1287 (vm_offset_t)buf + len <= bpage->datavaddr + 1288 bpage->datacount) 1289 return (1); 1290 } 1291 return (0); 1292 1293} 1294 1295void 1296_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1297{ 1298 struct mbuf *m; 1299 struct uio *uio; 1300 int resid; 1301 struct iovec *iov; 1302 int bufaligned; 1303 1304 if (op == BUS_DMASYNC_POSTWRITE) 1305 return; 1306 if (map->flags & DMAMAP_COHERENT) 1307 goto drain; 1308 if (STAILQ_FIRST(&map->bpages)) 1309 _bus_dmamap_sync_bp(dmat, map, op); 1310 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 1311 bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); 1312 switch(map->flags & DMAMAP_TYPE_MASK) { 1313 case DMAMAP_LINEAR: 1314 if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) 1315 bus_dmamap_sync_buf(map->buffer, map->len, op, 1316 bufaligned); 1317 break; 1318 case DMAMAP_MBUF: 1319 m = map->buffer; 1320 while (m) { 1321 if (m->m_len > 0 && 1322 !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) 1323 bus_dmamap_sync_buf(m->m_data, m->m_len, op, 1324 bufaligned); 1325 m = m->m_next; 1326 } 1327 break; 1328 case DMAMAP_UIO: 1329 uio = map->buffer; 1330 iov = uio->uio_iov; 1331 resid = uio->uio_resid; 1332 for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1333 bus_size_t minlen = resid < iov[i].iov_len ? resid : 1334 iov[i].iov_len; 1335 if (minlen > 0) { 1336 if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, 1337 minlen)) 1338 bus_dmamap_sync_buf(iov[i].iov_base, 1339 minlen, op, bufaligned); 1340 resid -= minlen; 1341 } 1342 } 1343 break; 1344 default: 1345 break; 1346 } 1347 1348drain: 1349 1350 cpu_drain_writebuf(); 1351} 1352 1353static void 1354init_bounce_pages(void *dummy __unused) 1355{ 1356 1357 total_bpages = 0; 1358 STAILQ_INIT(&bounce_zone_list); 1359 STAILQ_INIT(&bounce_map_waitinglist); 1360 STAILQ_INIT(&bounce_map_callbacklist); 1361 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1362} 1363SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1364 1365static struct sysctl_ctx_list * 1366busdma_sysctl_tree(struct bounce_zone *bz) 1367{ 1368 return (&bz->sysctl_tree); 1369} 1370 1371static struct sysctl_oid * 1372busdma_sysctl_tree_top(struct bounce_zone *bz) 1373{ 1374 return (bz->sysctl_tree_top); 1375} 1376 1377static int 1378alloc_bounce_zone(bus_dma_tag_t dmat) 1379{ 1380 struct bounce_zone *bz; 1381 1382 /* Check to see if we already have a suitable zone */ 1383 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1384 if ((dmat->alignment <= bz->alignment) 1385 && (dmat->lowaddr >= bz->lowaddr)) { 1386 dmat->bounce_zone = bz; 1387 return (0); 1388 } 1389 } 1390 1391 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1392 M_NOWAIT | M_ZERO)) == NULL) 1393 return (ENOMEM); 1394 1395 STAILQ_INIT(&bz->bounce_page_list); 1396 bz->free_bpages = 0; 1397 bz->reserved_bpages = 0; 1398 bz->active_bpages = 0; 1399 bz->lowaddr = dmat->lowaddr; 1400 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1401 bz->map_count = 0; 1402 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1403 busdma_zonecount++; 1404 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1405 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1406 dmat->bounce_zone = bz; 1407 1408 sysctl_ctx_init(&bz->sysctl_tree); 1409 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1410 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1411 CTLFLAG_RD, 0, ""); 1412 if (bz->sysctl_tree_top == NULL) { 1413 sysctl_ctx_free(&bz->sysctl_tree); 1414 return (0); /* XXX error code? */ 1415 } 1416 1417 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1418 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1419 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1420 "Total bounce pages"); 1421 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1422 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1423 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1424 "Free bounce pages"); 1425 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1426 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1427 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1428 "Reserved bounce pages"); 1429 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1430 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1431 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1432 "Active bounce pages"); 1433 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1434 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1435 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1436 "Total bounce requests"); 1437 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1438 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1439 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1440 "Total bounce requests that were deferred"); 1441 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1442 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1443 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1444 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1445 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1446 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1447 1448 return (0); 1449} 1450 1451static int 1452alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1453{ 1454 struct bounce_zone *bz; 1455 int count; 1456 1457 bz = dmat->bounce_zone; 1458 count = 0; 1459 while (numpages > 0) { 1460 struct bounce_page *bpage; 1461 1462 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1463 M_NOWAIT | M_ZERO); 1464 1465 if (bpage == NULL) 1466 break; 1467 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1468 M_NOWAIT, 0ul, 1469 bz->lowaddr, 1470 PAGE_SIZE, 1471 0); 1472 if (bpage->vaddr == 0) { 1473 free(bpage, M_DEVBUF); 1474 break; 1475 } 1476 bpage->busaddr = pmap_kextract(bpage->vaddr); 1477 bpage->vaddr_nocache = (vm_offset_t)arm_remap_nocache( 1478 (void *)bpage->vaddr, PAGE_SIZE); 1479 mtx_lock(&bounce_lock); 1480 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1481 total_bpages++; 1482 bz->total_bpages++; 1483 bz->free_bpages++; 1484 mtx_unlock(&bounce_lock); 1485 count++; 1486 numpages--; 1487 } 1488 return (count); 1489} 1490 1491static int 1492reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1493{ 1494 struct bounce_zone *bz; 1495 int pages; 1496 1497 mtx_assert(&bounce_lock, MA_OWNED); 1498 bz = dmat->bounce_zone; 1499 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1500 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1501 return (map->pagesneeded - (map->pagesreserved + pages)); 1502 bz->free_bpages -= pages; 1503 bz->reserved_bpages += pages; 1504 map->pagesreserved += pages; 1505 pages = map->pagesneeded - map->pagesreserved; 1506 1507 return (pages); 1508} 1509 1510static bus_addr_t 1511add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1512 bus_size_t size) 1513{ 1514 struct bounce_zone *bz; 1515 struct bounce_page *bpage; 1516 1517 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1518 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 1519 1520 bz = dmat->bounce_zone; 1521 if (map->pagesneeded == 0) 1522 panic("add_bounce_page: map doesn't need any pages"); 1523 map->pagesneeded--; 1524 1525 if (map->pagesreserved == 0) 1526 panic("add_bounce_page: map doesn't need any pages"); 1527 map->pagesreserved--; 1528 1529 mtx_lock(&bounce_lock); 1530 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1531 if (bpage == NULL) 1532 panic("add_bounce_page: free page list is empty"); 1533 1534 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1535 bz->reserved_bpages--; 1536 bz->active_bpages++; 1537 mtx_unlock(&bounce_lock); 1538 1539 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1540 /* Page offset needs to be preserved. */ 1541 bpage->vaddr |= vaddr & PAGE_MASK; 1542 bpage->busaddr |= vaddr & PAGE_MASK; 1543 } 1544 bpage->datavaddr = vaddr; 1545 bpage->datacount = size; 1546 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1547 return (bpage->busaddr); 1548} 1549 1550static void 1551free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1552{ 1553 struct bus_dmamap *map; 1554 struct bounce_zone *bz; 1555 1556 bz = dmat->bounce_zone; 1557 bpage->datavaddr = 0; 1558 bpage->datacount = 0; 1559 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1560 /* 1561 * Reset the bounce page to start at offset 0. Other uses 1562 * of this bounce page may need to store a full page of 1563 * data and/or assume it starts on a page boundary. 1564 */ 1565 bpage->vaddr &= ~PAGE_MASK; 1566 bpage->busaddr &= ~PAGE_MASK; 1567 } 1568 1569 mtx_lock(&bounce_lock); 1570 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1571 bz->free_bpages++; 1572 bz->active_bpages--; 1573 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1574 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1575 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1576 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1577 map, links); 1578 busdma_swi_pending = 1; 1579 bz->total_deferred++; 1580 swi_sched(vm_ih, 0); 1581 } 1582 } 1583 mtx_unlock(&bounce_lock); 1584} 1585 1586void 1587busdma_swi(void) 1588{ 1589 bus_dma_tag_t dmat; 1590 struct bus_dmamap *map; 1591 1592 mtx_lock(&bounce_lock); 1593 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1594 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1595 mtx_unlock(&bounce_lock); 1596 dmat = map->dmat; 1597 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1598 bus_dmamap_load(map->dmat, map, map->buffer, map->len, 1599 map->callback, map->callback_arg, /*flags*/0); 1600 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1601 mtx_lock(&bounce_lock); 1602 } 1603 mtx_unlock(&bounce_lock); 1604} 1605