busdma_machdep-v4.c revision 143294
1238104Sdes/*- 2238104Sdes * Copyright (c) 2004 Olivier Houchard 3238104Sdes * Copyright (c) 2002 Peter Grehan 4238104Sdes * Copyright (c) 1997, 1998 Justin T. Gibbs. 5238104Sdes * All rights reserved. 6238104Sdes * 7238104Sdes * Redistribution and use in source and binary forms, with or without 8238104Sdes * modification, are permitted provided that the following conditions 9269257Sdes * are met: 10238104Sdes * 1. Redistributions of source code must retain the above copyright 11238104Sdes * notice, this list of conditions, and the following disclaimer, 12238104Sdes * without modification, immediately at the beginning of the file. 13238104Sdes * 2. The name of the author may not be used to endorse or promote products 14238104Sdes * derived from this software without specific prior written permission. 15238104Sdes * 16269257Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17238104Sdes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18269257Sdes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19269257Sdes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 20269257Sdes * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21269257Sdes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22238104Sdes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23238104Sdes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24238104Sdes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25238104Sdes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26238104Sdes * SUCH DAMAGE. 27238104Sdes * 28238104Sdes * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred 29238104Sdes */ 30238104Sdes 31238104Sdes#include <sys/cdefs.h> 32238104Sdes__FBSDID("$FreeBSD: head/sys/arm/arm/busdma_machdep.c 143294 2005-03-08 14:49:05Z mux $"); 33238104Sdes 34238104Sdes/* 35238104Sdes * MacPPC bus dma support routines 36238104Sdes */ 37238104Sdes 38238104Sdes#define _ARM32_BUS_DMA_PRIVATE 39238104Sdes#include <sys/param.h> 40238104Sdes#include <sys/systm.h> 41238104Sdes#include <sys/malloc.h> 42238104Sdes#include <sys/bus.h> 43238104Sdes#include <sys/interrupt.h> 44238104Sdes#include <sys/lock.h> 45238104Sdes#include <sys/proc.h> 46238104Sdes#include <sys/mutex.h> 47238104Sdes#include <sys/mbuf.h> 48238104Sdes#include <sys/uio.h> 49238104Sdes#include <sys/ktr.h> 50238104Sdes 51238104Sdes#include <vm/vm.h> 52238104Sdes#include <vm/vm_page.h> 53238104Sdes#include <vm/vm_map.h> 54238104Sdes 55238104Sdes#include <machine/atomic.h> 56238104Sdes#include <machine/bus.h> 57238104Sdes#include <machine/cpufunc.h> 58238104Sdes 59238104Sdesstruct bus_dma_tag { 60238104Sdes bus_dma_tag_t parent; 61238104Sdes bus_size_t alignment; 62238104Sdes bus_size_t boundary; 63238104Sdes bus_addr_t lowaddr; 64238104Sdes bus_addr_t highaddr; 65238104Sdes bus_dma_filter_t *filter; 66238104Sdes void *filterarg; 67238104Sdes bus_size_t maxsize; 68238104Sdes u_int nsegments; 69246854Sdes bus_size_t maxsegsz; 70246854Sdes int flags; 71238104Sdes int ref_count; 72246854Sdes int map_count; 73238104Sdes bus_dma_lock_t *lockfunc; 74238104Sdes void *lockfuncarg; 75238104Sdes /* 76269257Sdes * DMA range for this tag. If the page doesn't fall within 77238104Sdes * one of these ranges, an error is returned. The caller 78238104Sdes * may then decide what to do with the transfer. If the 79238104Sdes * range pointer is NULL, it is ignored. 80238104Sdes */ 81238104Sdes struct arm32_dma_range *ranges; 82238104Sdes int _nranges; 83238104Sdes}; 84238104Sdes 85238104Sdes#define DMAMAP_LINEAR 0x1 86238104Sdes#define DMAMAP_MBUF 0x2 87238104Sdes#define DMAMAP_UIO 0x4 88238104Sdes#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) 89238104Sdes#define DMAMAP_COHERENT 0x8 90238104Sdesstruct bus_dmamap { 91238104Sdes bus_dma_tag_t dmat; 92238104Sdes int flags; 93238104Sdes void *buffer; 94238104Sdes int len; 95238104Sdes}; 96238104Sdes 97238104Sdes/* 98238104Sdes * Check to see if the specified page is in an allowed DMA range. 99238104Sdes */ 100238104Sdes 101238104Sdesstatic __inline int 102238104Sdesbus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 103238104Sdes bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 104238104Sdes int flags, vm_offset_t *lastaddrp, int *segp); 105238104Sdes 106238104Sdesstatic __inline struct arm32_dma_range * 107238104Sdes_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges, 108238104Sdes bus_addr_t curaddr) 109238104Sdes{ 110238104Sdes struct arm32_dma_range *dr; 111238104Sdes int i; 112238104Sdes 113238104Sdes for (i = 0, dr = ranges; i < nranges; i++, dr++) { 114238104Sdes if (curaddr >= dr->dr_sysbase && 115238104Sdes round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 116238104Sdes return (dr); 117238104Sdes } 118238104Sdes 119238104Sdes return (NULL); 120238104Sdes} 121246854Sdes/* 122238104Sdes * Convenience function for manipulating driver locks from busdma (during 123238104Sdes * busdma_swi, for example). Drivers that don't provide their own locks 124238104Sdes * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 125238104Sdes * non-mutex locking scheme don't have to use this at all. 126238104Sdes */ 127238104Sdesvoid 128246854Sdesbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 129238104Sdes{ 130238104Sdes struct mtx *dmtx; 131238104Sdes 132238104Sdes dmtx = (struct mtx *)arg; 133238104Sdes switch (op) { 134238104Sdes case BUS_DMA_LOCK: 135238104Sdes mtx_lock(dmtx); 136238104Sdes break; 137238104Sdes case BUS_DMA_UNLOCK: 138238104Sdes mtx_unlock(dmtx); 139238104Sdes break; 140238104Sdes default: 141238104Sdes panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 142238104Sdes } 143238104Sdes} 144238104Sdes 145238104Sdes/* 146238104Sdes * dflt_lock should never get called. It gets put into the dma tag when 147238104Sdes * lockfunc == NULL, which is only valid if the maps that are associated 148246854Sdes * with the tag are meant to never be defered. 149238104Sdes * XXX Should have a way to identify which driver is responsible here. 150238104Sdes */ 151238104Sdesstatic void 152238104Sdesdflt_lock(void *arg, bus_dma_lock_op_t op) 153238104Sdes{ 154238104Sdes#ifdef INVARIANTS 155246854Sdes panic("driver error: busdma dflt_lock called"); 156238104Sdes#else 157238104Sdes printf("DRIVER_ERROR: busdma dflt_lock called\n"); 158238104Sdes#endif 159269257Sdes} 160238104Sdes 161238104Sdes/* 162238104Sdes * Allocate a device specific dma_tag. 163238104Sdes */ 164238104Sdes#define SEG_NB 1024 165238104Sdes 166238104Sdesint 167238104Sdesbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 168238104Sdes bus_size_t boundary, bus_addr_t lowaddr, 169238104Sdes bus_addr_t highaddr, bus_dma_filter_t *filter, 170238104Sdes void *filterarg, bus_size_t maxsize, int nsegments, 171238104Sdes bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 172238104Sdes void *lockfuncarg, bus_dma_tag_t *dmat) 173269257Sdes{ 174269257Sdes bus_dma_tag_t newtag; 175269257Sdes int error = 0; 176269257Sdes /* Return a NULL tag on failure */ 177269257Sdes *dmat = NULL; 178269257Sdes 179269257Sdes newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 180269257Sdes if (newtag == NULL) { 181269257Sdes CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 182269257Sdes __func__, newtag, 0, error); 183238104Sdes return (ENOMEM); 184238104Sdes } 185238104Sdes 186238104Sdes newtag->parent = parent; 187238104Sdes newtag->alignment = alignment; 188238104Sdes newtag->boundary = boundary; 189238104Sdes newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 190238104Sdes newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); 191238104Sdes newtag->filter = filter; 192238104Sdes newtag->filterarg = filterarg; 193238104Sdes newtag->maxsize = maxsize; 194238104Sdes newtag->nsegments = nsegments; 195238104Sdes newtag->maxsegsz = maxsegsz; 196238104Sdes newtag->flags = flags; 197238104Sdes newtag->ref_count = 1; /* Count ourself */ 198238104Sdes newtag->map_count = 0; 199238104Sdes newtag->ranges = bus_dma_get_range(); 200238104Sdes newtag->_nranges = bus_dma_get_range_nb(); 201238104Sdes if (lockfunc != NULL) { 202238104Sdes newtag->lockfunc = lockfunc; 203238104Sdes newtag->lockfuncarg = lockfuncarg; 204238104Sdes } else { 205238104Sdes newtag->lockfunc = dflt_lock; 206238104Sdes newtag->lockfuncarg = NULL; 207238104Sdes } 208238104Sdes /* 209238104Sdes * Take into account any restrictions imposed by our parent tag 210238104Sdes */ 211238104Sdes if (parent != NULL) { 212238104Sdes newtag->lowaddr = min(parent->lowaddr, newtag->lowaddr); 213238104Sdes newtag->highaddr = max(parent->highaddr, newtag->highaddr); 214238104Sdes if (newtag->boundary == 0) 215238104Sdes newtag->boundary = parent->boundary; 216238104Sdes else if (parent->boundary != 0) 217238104Sdes newtag->boundary = min(parent->boundary, 218238104Sdes newtag->boundary); 219238104Sdes if (newtag->filter == NULL) { 220238104Sdes /* 221238104Sdes * Short circuit looking at our parent directly 222238104Sdes * since we have encapsulated all of its information 223238104Sdes */ 224238104Sdes newtag->filter = parent->filter; 225238104Sdes newtag->filterarg = parent->filterarg; 226238104Sdes newtag->parent = parent->parent; 227238104Sdes } 228238104Sdes if (newtag->parent != NULL) 229238104Sdes atomic_add_int(&parent->ref_count, 1); 230238104Sdes } 231238104Sdes 232238104Sdes *dmat = newtag; 233238104Sdes CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 234238104Sdes __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 235238104Sdes 236238104Sdes return (error); 237238104Sdes} 238238104Sdes 239238104Sdesint 240238104Sdesbus_dma_tag_destroy(bus_dma_tag_t dmat) 241238104Sdes{ 242238104Sdes#ifdef KTR 243238104Sdes bus_dma_tag_t dmat_copy = dmat; 244238104Sdes#endif 245238104Sdes 246238104Sdes if (dmat != NULL) { 247238104Sdes 248238104Sdes if (dmat->map_count != 0) 249238104Sdes return (EBUSY); 250238104Sdes 251238104Sdes while (dmat != NULL) { 252238104Sdes bus_dma_tag_t parent; 253238104Sdes 254238104Sdes parent = dmat->parent; 255238104Sdes atomic_subtract_int(&dmat->ref_count, 1); 256238104Sdes if (dmat->ref_count == 0) { 257238104Sdes free(dmat, M_DEVBUF); 258238104Sdes /* 259238104Sdes * Last reference count, so 260238104Sdes * release our reference 261238104Sdes * count on our parent. 262238104Sdes */ 263238104Sdes dmat = parent; 264238104Sdes } else 265238104Sdes dmat = NULL; 266238104Sdes } 267238104Sdes } 268238104Sdes CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy); 269238104Sdes 270238104Sdes return (0); 271238104Sdes} 272269257Sdes 273269257Sdes/* 274269257Sdes * Allocate a handle for mapping from kva/uva/physical 275269257Sdes * address space into bus device space. 276269257Sdes */ 277269257Sdesint 278269257Sdesbus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 279269257Sdes{ 280269257Sdes bus_dmamap_t newmap; 281269257Sdes#ifdef KTR 282269257Sdes int error = 0; 283269257Sdes#endif 284269257Sdes 285269257Sdes newmap = malloc(sizeof(*newmap), M_DEVBUF, M_NOWAIT | M_ZERO); 286269257Sdes if (newmap == NULL) { 287269257Sdes CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); 288269257Sdes return (ENOMEM); 289269257Sdes } 290269257Sdes *mapp = newmap; 291269257Sdes newmap->dmat = dmat; 292269257Sdes newmap->flags = 0; 293269257Sdes dmat->map_count++; 294269257Sdes 295269257Sdes CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 296238104Sdes __func__, dmat, dmat->flags, error); 297238104Sdes 298238104Sdes return (0); 299238104Sdes} 300238104Sdes 301238104Sdes/* 302238104Sdes * Destroy a handle for mapping from kva/uva/physical 303238104Sdes * address space into bus device space. 304238104Sdes */ 305238104Sdesint 306238104Sdesbus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 307238104Sdes{ 308238104Sdes 309238104Sdes free(map, M_DEVBUF); 310238104Sdes dmat->map_count--; 311238104Sdes CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 312238104Sdes return (0); 313238104Sdes} 314238104Sdes 315238104Sdes/* 316238104Sdes * Allocate a piece of memory that can be efficiently mapped into 317238104Sdes * bus device space based on the constraints lited in the dma tag. 318238104Sdes * A dmamap to for use with dmamap_load is also allocated. 319238104Sdes */ 320238104Sdesint 321238104Sdesbus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 322238104Sdes bus_dmamap_t *mapp) 323238104Sdes{ 324238104Sdes bus_dmamap_t newmap = NULL; 325238104Sdes 326238104Sdes int mflags; 327238104Sdes 328238104Sdes if (flags & BUS_DMA_NOWAIT) 329238104Sdes mflags = M_NOWAIT; 330238104Sdes else 331238104Sdes mflags = M_WAITOK; 332238104Sdes if (flags & BUS_DMA_ZERO) 333238104Sdes mflags |= M_ZERO; 334238104Sdes 335238104Sdes if (!*mapp) { 336238104Sdes newmap = malloc(sizeof(*newmap), M_DEVBUF, M_NOWAIT | M_ZERO); 337238104Sdes if (newmap == NULL) { 338238104Sdes CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 339238104Sdes __func__, dmat, dmat->flags, ENOMEM); 340238104Sdes return (ENOMEM); 341238104Sdes } 342238104Sdes dmat->map_count++; 343238104Sdes newmap->flags = 0; 344238104Sdes *mapp = newmap; 345238104Sdes newmap->dmat = dmat; 346238104Sdes } 347238104Sdes 348238104Sdes if (dmat->maxsize <= PAGE_SIZE) { 349238104Sdes *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 350238104Sdes } else { 351238104Sdes /* 352238104Sdes * XXX Use Contigmalloc until it is merged into this facility 353238104Sdes * and handles multi-seg allocations. Nobody is doing 354238104Sdes * multi-seg allocations yet though. 355238104Sdes */ 356238104Sdes *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 357238104Sdes 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 358238104Sdes dmat->boundary); 359238104Sdes } 360238104Sdes if (*vaddr == NULL) { 361269257Sdes if (newmap != NULL) { 362269257Sdes free(newmap, M_DEVBUF); 363269257Sdes dmat->map_count--; 364269257Sdes } 365269257Sdes *mapp = NULL; 366269257Sdes return (ENOMEM); 367269257Sdes } 368269257Sdes return (0); 369269257Sdes} 370269257Sdes 371269257Sdes/* 372269257Sdes * Free a piece of memory and it's allocated dmamap, that was allocated 373269257Sdes * via bus_dmamem_alloc. Make the same choice for free/contigfree. 374269257Sdes */ 375269257Sdesvoid 376269257Sdesbus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 377269257Sdes{ 378269257Sdes if (dmat->maxsize <= PAGE_SIZE) 379269257Sdes free(vaddr, M_DEVBUF); 380269257Sdes else { 381269257Sdes contigfree(vaddr, dmat->maxsize, M_DEVBUF); 382269257Sdes } 383269257Sdes dmat->map_count--; 384269257Sdes free(map, M_DEVBUF); 385269257Sdes CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 386269257Sdes} 387269257Sdes 388269257Sdes/* 389269257Sdes * Utility function to load a linear buffer. lastaddrp holds state 390269257Sdes * between invocations (for multiple-buffer loads). segp contains 391269257Sdes * the starting segment on entrance, and the ending segment on exit. 392269257Sdes * first indicates if this is the first invocation of this function. 393269257Sdes */ 394269257Sdesstatic int __inline 395269257Sdesbus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, 396269257Sdes bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, 397269257Sdes int flags, vm_offset_t *lastaddrp, int *segp) 398269257Sdes{ 399269257Sdes bus_size_t sgsize; 400269257Sdes bus_addr_t curaddr, lastaddr, baddr, bmask; 401269257Sdes vm_offset_t vaddr = (vm_offset_t)buf; 402269257Sdes int seg; 403269257Sdes int error = 0; 404269257Sdes pd_entry_t *pde; 405269257Sdes pt_entry_t pte; 406269257Sdes pt_entry_t *ptep; 407269257Sdes 408269257Sdes lastaddr = *lastaddrp; 409269257Sdes bmask = ~(dmat->boundary - 1); 410269257Sdes 411269257Sdes CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " 412269257Sdes "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); 413269257Sdes 414269257Sdes for (seg = *segp; buflen > 0 ; ) { 415269257Sdes /* 416269257Sdes * Get the physical address for this segment. 417238104Sdes * 418238104Sdes * XXX Don't support checking for coherent mappings 419238104Sdes * XXX in user address space. 420246854Sdes */ 421269257Sdes if (__predict_true(pmap == pmap_kernel())) { 422246854Sdes (void) pmap_get_pde_pte(pmap, vaddr, &pde, &ptep); 423238104Sdes if (__predict_false(pmap_pde_section(pde))) { 424238104Sdes curaddr = (*pde & L1_S_FRAME) | 425238104Sdes (vaddr & L1_S_OFFSET); 426238104Sdes if (*pde & L1_S_CACHE_MASK) { 427238104Sdes map->flags &= 428238104Sdes ~DMAMAP_COHERENT; 429238104Sdes } 430238104Sdes } else { 431238104Sdes pte = *ptep; 432238104Sdes KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV, 433238104Sdes ("INV type")); 434238104Sdes if (__predict_false((pte & L2_TYPE_MASK) 435238104Sdes == L2_TYPE_L)) { 436238104Sdes curaddr = (pte & L2_L_FRAME) | 437238104Sdes (vaddr & L2_L_OFFSET); 438238104Sdes if (pte & L2_L_CACHE_MASK) { 439238104Sdes map->flags &= 440238104Sdes ~DMAMAP_COHERENT; 441238104Sdes 442238104Sdes } 443238104Sdes } else { 444238104Sdes curaddr = (pte & L2_S_FRAME) | 445238104Sdes (vaddr & L2_S_OFFSET); 446238104Sdes if (pte & L2_S_CACHE_MASK) { 447238104Sdes map->flags &= 448238104Sdes ~DMAMAP_COHERENT; 449238104Sdes } 450238104Sdes } 451238104Sdes } 452238104Sdes } else { 453238104Sdes curaddr = pmap_extract(pmap, vaddr); 454238104Sdes map->flags &= ~DMAMAP_COHERENT; 455238104Sdes } 456238104Sdes 457238104Sdes if (dmat->ranges) { 458238104Sdes struct arm32_dma_range *dr; 459238104Sdes 460238104Sdes dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, 461238104Sdes curaddr); 462238104Sdes if (dr == NULL) 463238104Sdes return (EINVAL); 464238104Sdes /* 465238104Sdes * In a valid DMA range. Translate the physical 466238104Sdes * memory address to an address in the DMA window. 467238104Sdes */ 468238104Sdes curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; 469238104Sdes 470238104Sdes } 471238104Sdes /* 472238104Sdes * Compute the segment size, and adjust counts. 473238104Sdes */ 474238104Sdes sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 475238104Sdes if (buflen < sgsize) 476238104Sdes sgsize = buflen; 477238104Sdes 478238104Sdes /* 479238104Sdes * Make sure we don't cross any boundaries. 480238104Sdes */ 481238104Sdes if (dmat->boundary > 0) { 482238104Sdes baddr = (curaddr + dmat->boundary) & bmask; 483238104Sdes if (sgsize > (baddr - curaddr)) 484238104Sdes sgsize = (baddr - curaddr); 485238104Sdes } 486238104Sdes 487238104Sdes /* 488238104Sdes * Insert chunk into a segment, coalescing with 489238104Sdes * the previous segment if possible. 490238104Sdes */ 491238104Sdes if (seg >= 0 && curaddr == lastaddr && 492238104Sdes (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 493238104Sdes (dmat->boundary == 0 || 494238104Sdes (segs[seg].ds_addr & bmask) == 495238104Sdes (curaddr & bmask))) { 496238104Sdes segs[seg].ds_len += sgsize; 497238104Sdes goto segdone; 498238104Sdes } else { 499238104Sdes if (++seg >= dmat->nsegments) 500238104Sdes break; 501238104Sdes segs[seg].ds_addr = curaddr; 502238104Sdes segs[seg].ds_len = sgsize; 503238104Sdes } 504238104Sdes if (error) 505238104Sdes break; 506238104Sdessegdone: 507238104Sdes lastaddr = curaddr + sgsize; 508238104Sdes vaddr += sgsize; 509238104Sdes buflen -= sgsize; 510238104Sdes } 511238104Sdes 512238104Sdes *segp = seg; 513238104Sdes *lastaddrp = lastaddr; 514238104Sdes 515238104Sdes /* 516238104Sdes * Did we fit? 517238104Sdes */ 518238104Sdes if (buflen != 0) 519238104Sdes error = EFBIG; /* XXX better return value here? */ 520238104Sdes return (error); 521238104Sdes} 522238104Sdes 523269257Sdes/* 524238104Sdes * Map the buffer buf into bus space using the dmamap map. 525238104Sdes */ 526238104Sdesint 527238104Sdesbus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 528238104Sdes bus_size_t buflen, bus_dmamap_callback_t *callback, 529238104Sdes void *callback_arg, int flags) 530238104Sdes{ 531238104Sdes vm_offset_t lastaddr = 0; 532238104Sdes int error, nsegs = -1; 533238104Sdes#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT 534238104Sdes bus_dma_segment_t dm_segments[dmat->nsegments]; 535238104Sdes#else 536238104Sdes bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 537238104Sdes#endif 538238104Sdes 539238104Sdes map->flags &= ~DMAMAP_TYPE_MASK; 540238104Sdes map->flags |= DMAMAP_LINEAR|DMAMAP_COHERENT; 541238104Sdes map->buffer = buf; 542238104Sdes map->len = buflen; 543238104Sdes error = bus_dmamap_load_buffer(dmat, 544238104Sdes dm_segments, map, buf, buflen, kernel_pmap, 545238104Sdes flags, &lastaddr, &nsegs); 546238104Sdes if (error) 547238104Sdes (*callback)(callback_arg, NULL, 0, error); 548238104Sdes else 549246854Sdes (*callback)(callback_arg, dm_segments, nsegs + 1, error); 550246854Sdes 551246854Sdes CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 552246854Sdes __func__, dmat, dmat->flags, nsegs + 1, error); 553246854Sdes 554238104Sdes return (0); 555269257Sdes} 556269257Sdes 557269257Sdes/* 558238104Sdes * Like bus_dmamap_load(), but for mbufs. 559238104Sdes */ 560238104Sdesint 561238104Sdesbus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 562238104Sdes bus_dmamap_callback2_t *callback, void *callback_arg, 563238104Sdes int flags) 564238104Sdes{ 565238104Sdes#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT 566238104Sdes bus_dma_segment_t dm_segments[dmat->nsegments]; 567238104Sdes#else 568238104Sdes bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 569238104Sdes#endif 570238104Sdes int nsegs = -1, error = 0; 571238104Sdes 572238104Sdes M_ASSERTPKTHDR(m0); 573238104Sdes 574238104Sdes map->flags &= ~DMAMAP_TYPE_MASK; 575238104Sdes map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT; 576238104Sdes map->buffer = m0; 577238104Sdes if (m0->m_pkthdr.len <= dmat->maxsize) { 578238104Sdes vm_offset_t lastaddr = 0; 579238104Sdes struct mbuf *m; 580238104Sdes 581238104Sdes for (m = m0; m != NULL && error == 0; m = m->m_next) { 582238104Sdes if (m->m_len > 0) 583238104Sdes error = bus_dmamap_load_buffer(dmat, 584238104Sdes dm_segments, map, m->m_data, m->m_len, 585238104Sdes pmap_kernel(), flags, &lastaddr, &nsegs); 586238104Sdes } 587238104Sdes } else { 588238104Sdes error = EINVAL; 589238104Sdes } 590238104Sdes 591238104Sdes if (error) { 592238104Sdes /* 593269257Sdes * force "no valid mappings" on error in callback. 594269257Sdes */ 595269257Sdes (*callback)(callback_arg, dm_segments, 0, 0, error); 596269257Sdes } else { 597269257Sdes (*callback)(callback_arg, dm_segments, nsegs + 1, 598269257Sdes m0->m_pkthdr.len, error); 599269257Sdes } 600269257Sdes CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 601269257Sdes __func__, dmat, dmat->flags, error, nsegs + 1); 602269257Sdes 603269257Sdes return (error); 604238104Sdes} 605238104Sdes 606238104Sdesint 607238104Sdesbus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 608238104Sdes struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 609238104Sdes int flags) 610238104Sdes{ 611238104Sdes int error = 0; 612238104Sdes M_ASSERTPKTHDR(m0); 613238104Sdes 614238104Sdes flags |= BUS_DMA_NOWAIT; 615238104Sdes *nsegs = -1; 616238104Sdes map->flags &= ~DMAMAP_TYPE_MASK; 617238104Sdes map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT; 618238104Sdes map->buffer = m0; 619238104Sdes if (m0->m_pkthdr.len <= dmat->maxsize) { 620238104Sdes vm_offset_t lastaddr = 0; 621238104Sdes struct mbuf *m; 622238104Sdes 623238104Sdes for (m = m0; m != NULL && error == 0; m = m->m_next) { 624238104Sdes if (m->m_len > 0) { 625238104Sdes error = bus_dmamap_load_buffer(dmat, segs, map, 626246854Sdes m->m_data, m->m_len, 627246854Sdes pmap_kernel(), flags, &lastaddr, 628246854Sdes nsegs); 629246854Sdes } 630246854Sdes } 631246854Sdes } else { 632246854Sdes error = EINVAL; 633246854Sdes } 634246854Sdes 635246854Sdes /* XXX FIXME: Having to increment nsegs is really annoying */ 636246854Sdes ++*nsegs; 637246854Sdes CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 638246854Sdes __func__, dmat, dmat->flags, error, *nsegs); 639246854Sdes return (error); 640246854Sdes} 641246854Sdes 642246854Sdes/* 643246854Sdes * Like bus_dmamap_load(), but for uios. 644246854Sdes */ 645246854Sdesint 646246854Sdesbus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 647246854Sdes bus_dmamap_callback2_t *callback, void *callback_arg, 648246854Sdes int flags) 649246854Sdes{ 650246854Sdes vm_offset_t lastaddr; 651246854Sdes#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT 652246854Sdes bus_dma_segment_t dm_segments[dmat->nsegments]; 653246854Sdes#else 654246854Sdes bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 655246854Sdes#endif 656246854Sdes int nsegs, i, error; 657246854Sdes bus_size_t resid; 658246854Sdes struct iovec *iov; 659246854Sdes struct pmap *pmap; 660246854Sdes 661246854Sdes resid = uio->uio_resid; 662246854Sdes iov = uio->uio_iov; 663246854Sdes map->flags &= ~DMAMAP_TYPE_MASK; 664246854Sdes map->flags |= DMAMAP_UIO|DMAMAP_COHERENT; 665238104Sdes map->buffer = uio; 666238104Sdes 667238104Sdes if (uio->uio_segflg == UIO_USERSPACE) { 668238104Sdes KASSERT(uio->uio_td != NULL, 669238104Sdes ("bus_dmamap_load_uio: USERSPACE but no proc")); 670238104Sdes pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 671238104Sdes } else 672238104Sdes pmap = kernel_pmap; 673238104Sdes 674238104Sdes error = 0; 675238104Sdes nsegs = -1; 676238104Sdes for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 677238104Sdes /* 678238104Sdes * Now at the first iovec to load. Load each iovec 679238104Sdes * until we have exhausted the residual count. 680238104Sdes */ 681238104Sdes bus_size_t minlen = 682238104Sdes resid < iov[i].iov_len ? resid : iov[i].iov_len; 683238104Sdes caddr_t addr = (caddr_t) iov[i].iov_base; 684238104Sdes 685238104Sdes if (minlen > 0) { 686238104Sdes error = bus_dmamap_load_buffer(dmat, dm_segments, map, 687238104Sdes addr, minlen, pmap, flags, &lastaddr, &nsegs); 688238104Sdes 689238104Sdes resid -= minlen; 690238104Sdes } 691238104Sdes } 692238104Sdes 693238104Sdes if (error) { 694238104Sdes /* 695238104Sdes * force "no valid mappings" on error in callback. 696238104Sdes */ 697238104Sdes (*callback)(callback_arg, dm_segments, 0, 0, error); 698238104Sdes } else { 699238104Sdes (*callback)(callback_arg, dm_segments, nsegs+1, 700238104Sdes uio->uio_resid, error); 701238104Sdes } 702238104Sdes 703238104Sdes CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 704238104Sdes __func__, dmat, dmat->flags, error, nsegs + 1); 705238104Sdes return (error); 706238104Sdes} 707238104Sdes 708238104Sdes/* 709238104Sdes * Release the mapping held by map. 710238104Sdes */ 711238104Sdesvoid 712238104Sdesbus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 713238104Sdes{ 714238104Sdes map->flags &= ~DMAMAP_TYPE_MASK; 715238104Sdes return; 716238104Sdes} 717238104Sdes 718238104Sdesstatic void 719238104Sdesbus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op) 720238104Sdes{ 721238104Sdes 722238104Sdes if (op & BUS_DMASYNC_POSTREAD || 723238104Sdes op == (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) { 724238104Sdes cpu_dcache_wbinv_range((vm_offset_t)buf, len); 725238104Sdes return; 726238104Sdes } 727238104Sdes if (op & BUS_DMASYNC_PREWRITE) 728238104Sdes cpu_dcache_wb_range((vm_offset_t)buf, len); 729238104Sdes if (op & BUS_DMASYNC_PREREAD) { 730238104Sdes if ((((vm_offset_t)buf | len) & arm_dcache_align_mask) == 0) 731238104Sdes cpu_dcache_inv_range((vm_offset_t)buf, len); 732238104Sdes else 733238104Sdes cpu_dcache_wbinv_range((vm_offset_t)buf, len); 734238104Sdes } 735238104Sdes} 736238104Sdes 737238104Sdesvoid 738238104Sdesbus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 739238104Sdes{ 740238104Sdes struct mbuf *m; 741238104Sdes struct uio *uio; 742238104Sdes int resid; 743238104Sdes struct iovec *iov; 744269257Sdes 745269257Sdes if (op == BUS_DMASYNC_POSTWRITE) 746269257Sdes return; 747269257Sdes if (map->flags & DMAMAP_COHERENT) 748269257Sdes return; 749269257Sdes CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); 750238104Sdes switch(map->flags & DMAMAP_TYPE_MASK) { 751238104Sdes case DMAMAP_LINEAR: 752238104Sdes bus_dmamap_sync_buf(map->buffer, map->len, op); 753238104Sdes break; 754238104Sdes case DMAMAP_MBUF: 755238104Sdes m = map->buffer; 756238104Sdes while (m) { 757238104Sdes bus_dmamap_sync_buf(m->m_data, m->m_len, op); 758238104Sdes m = m->m_next; 759238104Sdes } 760238104Sdes break; 761238104Sdes case DMAMAP_UIO: 762238104Sdes uio = map->buffer; 763238104Sdes iov = uio->uio_iov; 764238104Sdes resid = uio->uio_resid; 765238104Sdes for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 766238104Sdes bus_size_t minlen = resid < iov[i].iov_len ? resid : 767238104Sdes iov[i].iov_len; 768238104Sdes if (minlen > 0) { 769238104Sdes bus_dmamap_sync_buf(iov[i].iov_base, minlen, 770238104Sdes op); 771238104Sdes resid -= minlen; 772238104Sdes } 773238104Sdes } 774238104Sdes break; 775238104Sdes default: 776238104Sdes break; 777238104Sdes } 778238104Sdes cpu_drain_writebuf(); 779238104Sdes} 780238104Sdes