186228Stmm/*- 286228Stmm * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. 386228Stmm * All rights reserved. 486228Stmm * 586228Stmm * This code is derived from software contributed to The NetBSD Foundation 686228Stmm * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 786228Stmm * NASA Ames Research Center. 886228Stmm * 986228Stmm * Redistribution and use in source and binary forms, with or without 1086228Stmm * modification, are permitted provided that the following conditions 1186228Stmm * are met: 1286228Stmm * 1. Redistributions of source code must retain the above copyright 1386228Stmm * notice, this list of conditions and the following disclaimer. 1486228Stmm * 2. Redistributions in binary form must reproduce the above copyright 1586228Stmm * notice, this list of conditions and the following disclaimer in the 1686228Stmm * documentation and/or other materials provided with the distribution. 1786228Stmm * 1886228Stmm * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 1986228Stmm * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 2086228Stmm * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 2186228Stmm * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 2286228Stmm * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2386228Stmm * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2486228Stmm * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2586228Stmm * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2686228Stmm * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2786228Stmm * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 2886228Stmm * POSSIBILITY OF SUCH DAMAGE. 2986228Stmm */ 30139825Simp/*- 3186228Stmm * Copyright (c) 1992, 1993 3286228Stmm * The Regents of the University of California. All rights reserved. 3386228Stmm * 3486228Stmm * This software was developed by the Computer Systems Engineering group 3586228Stmm * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 3686228Stmm * contributed to Berkeley. 3786228Stmm * 3886228Stmm * Redistribution and use in source and binary forms, with or without 3986228Stmm * modification, are permitted provided that the following conditions 4086228Stmm * are met: 4186228Stmm * 1. Redistributions of source code must retain the above copyright 4286228Stmm * notice, this list of conditions and the following disclaimer. 4386228Stmm * 2. Redistributions in binary form must reproduce the above copyright 4486228Stmm * notice, this list of conditions and the following disclaimer in the 4586228Stmm * documentation and/or other materials provided with the distribution. 4686228Stmm * 4. Neither the name of the University nor the names of its contributors 4786228Stmm * may be used to endorse or promote products derived from this software 4886228Stmm * without specific prior written permission. 4986228Stmm * 5086228Stmm * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 5186228Stmm * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 5286228Stmm * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 5386228Stmm * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 5486228Stmm * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 5586228Stmm * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 5686228Stmm * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 5786228Stmm * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 5886228Stmm * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 5986228Stmm * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 6086228Stmm * SUCH DAMAGE. 6186228Stmm */ 62139825Simp/*- 6386228Stmm * Copyright (c) 1997, 1998 Justin T. Gibbs. 6486228Stmm * All rights reserved. 6586228Stmm * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>. All rights reserved. 6686228Stmm * 6786228Stmm * Redistribution and use in source and binary forms, with or without 6886228Stmm * modification, are permitted provided that the following conditions 6986228Stmm * are met: 7086228Stmm * 1. Redistributions of source code must retain the above copyright 7186228Stmm * notice, this list of conditions, and the following disclaimer, 7286228Stmm * without modification, immediately at the beginning of the file. 7386228Stmm * 2. The name of the author may not be used to endorse or promote products 7486228Stmm * derived from this software without specific prior written permission. 7586228Stmm * 7686228Stmm * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 7786228Stmm * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 7886228Stmm * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 7986228Stmm * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 8086228Stmm * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 8186228Stmm * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 8286228Stmm * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 8386228Stmm * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 8486228Stmm * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 8586228Stmm * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 8686228Stmm * SUCH DAMAGE. 8786228Stmm * 8886228Stmm * from: @(#)machdep.c 8.6 (Berkeley) 1/14/94 89219567Smarius * from: NetBSD: machdep.c,v 1.221 2008/04/28 20:23:37 martin Exp 9086228Stmm * and 91177565Smarius * from: FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.24 2001/08/15 9286228Stmm */ 9386228Stmm 94166096Smarius#include <sys/cdefs.h> 95166096Smarius__FBSDID("$FreeBSD$"); 96166096Smarius 9786228Stmm#include <sys/param.h> 9886228Stmm#include <sys/bus.h> 99112227Sjake#include <sys/lock.h> 10086228Stmm#include <sys/malloc.h> 101112215Smux#include <sys/mutex.h> 10286228Stmm#include <sys/proc.h> 103230687Smarius#include <sys/rman.h> 10491783Sjake#include <sys/smp.h> 10586228Stmm#include <sys/systm.h> 10686228Stmm 10786228Stmm#include <vm/vm.h> 10886228Stmm#include <vm/vm_extern.h> 10986228Stmm#include <vm/vm_kern.h> 11086228Stmm#include <vm/vm_page.h> 11186228Stmm#include <vm/vm_param.h> 112104486Ssam#include <vm/vm_map.h> 11386228Stmm 11486228Stmm#include <machine/asi.h> 115112436Smux#include <machine/atomic.h> 11686228Stmm#include <machine/bus.h> 11793070Stmm#include <machine/bus_private.h> 11886228Stmm#include <machine/cache.h> 11991783Sjake#include <machine/smp.h> 12091177Sjake#include <machine/tlb.h> 12186228Stmm 122119398Smarcelstatic void nexus_bus_barrier(bus_space_tag_t, bus_space_handle_t, 123119398Smarcel bus_size_t, bus_size_t, int); 124119398Smarcel 125177565Smarius/* ASIs for bus access */ 126177565Smariusconst int bus_type_asi[] = { 127166096Smarius ASI_PHYS_BYPASS_EC_WITH_EBIT, /* nexus */ 128166096Smarius ASI_PHYS_BYPASS_EC_WITH_EBIT, /* SBus */ 12986228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI configuration space */ 13086228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI memory space */ 13186228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI I/O space */ 13286228Stmm 0 13386228Stmm}; 13486228Stmm 135177565Smariusconst int bus_stream_asi[] = { 136166096Smarius ASI_PHYS_BYPASS_EC_WITH_EBIT, /* nexus */ 137166096Smarius ASI_PHYS_BYPASS_EC_WITH_EBIT, /* SBus */ 13886228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI configuration space */ 13986228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI memory space */ 14086228Stmm ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI I/O space */ 14186228Stmm 0 14286228Stmm}; 14386228Stmm 14486228Stmm/* 145117126Sscottl * Convenience function for manipulating driver locks from busdma (during 146117126Sscottl * busdma_swi, for example). Drivers that don't provide their own locks 147117126Sscottl * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 148117126Sscottl * non-mutex locking scheme don't have to use this at all. 149117126Sscottl */ 150117126Sscottlvoid 151117126Sscottlbusdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 152117126Sscottl{ 153117126Sscottl struct mtx *dmtx; 154117126Sscottl 155117126Sscottl dmtx = (struct mtx *)arg; 156117126Sscottl switch (op) { 157117126Sscottl case BUS_DMA_LOCK: 158117126Sscottl mtx_lock(dmtx); 159117126Sscottl break; 160117126Sscottl case BUS_DMA_UNLOCK: 161117126Sscottl mtx_unlock(dmtx); 162117126Sscottl break; 163117126Sscottl default: 164117126Sscottl panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 165117126Sscottl } 166117126Sscottl} 167117126Sscottl 168117126Sscottl/* 169117126Sscottl * dflt_lock should never get called. It gets put into the dma tag when 170117126Sscottl * lockfunc == NULL, which is only valid if the maps that are associated 171117126Sscottl * with the tag are meant to never be defered. 172117126Sscottl * XXX Should have a way to identify which driver is responsible here. 173117126Sscottl */ 174117126Sscottlstatic void 175117126Sscottldflt_lock(void *arg, bus_dma_lock_op_t op) 176117126Sscottl{ 177212676Smarius 178117126Sscottl panic("driver error: busdma dflt_lock called"); 179117126Sscottl} 180117126Sscottl 181117126Sscottl/* 18286228Stmm * Allocate a device specific dma_tag. 18386228Stmm */ 18486228Stmmint 18586228Stmmbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 18686228Stmm bus_size_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 18786228Stmm bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 188117126Sscottl int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 189117126Sscottl void *lockfuncarg, bus_dma_tag_t *dmat) 19086228Stmm{ 19193070Stmm bus_dma_tag_t newtag; 19286228Stmm 19386228Stmm /* Return a NULL tag on failure */ 19486228Stmm *dmat = NULL; 19586228Stmm 196167308Smarius /* Enforce the usage of BUS_GET_DMA_TAG(). */ 197167308Smarius if (parent == NULL) 198167308Smarius panic("%s: parent DMA tag NULL", __func__); 199167308Smarius 20086228Stmm newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 20186228Stmm if (newtag == NULL) 20286228Stmm return (ENOMEM); 20386228Stmm 204116541Stmm /* 205116541Stmm * The method table pointer and the cookie need to be taken over from 206167308Smarius * the parent. 207116541Stmm */ 208167308Smarius newtag->dt_cookie = parent->dt_cookie; 209167308Smarius newtag->dt_mt = parent->dt_mt; 210116541Stmm 211116541Stmm newtag->dt_parent = parent; 212108815Stmm newtag->dt_alignment = alignment; 213108815Stmm newtag->dt_boundary = boundary; 214108815Stmm newtag->dt_lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); 215108815Stmm newtag->dt_highaddr = trunc_page((vm_offset_t)highaddr) + 216108815Stmm (PAGE_SIZE - 1); 217108815Stmm newtag->dt_filter = filter; 218108815Stmm newtag->dt_filterarg = filterarg; 219108815Stmm newtag->dt_maxsize = maxsize; 220108815Stmm newtag->dt_nsegments = nsegments; 221108815Stmm newtag->dt_maxsegsz = maxsegsz; 222108815Stmm newtag->dt_flags = flags; 223108815Stmm newtag->dt_ref_count = 1; /* Count ourselves */ 224108815Stmm newtag->dt_map_count = 0; 22593070Stmm 226117126Sscottl if (lockfunc != NULL) { 227117126Sscottl newtag->dt_lockfunc = lockfunc; 228117126Sscottl newtag->dt_lockfuncarg = lockfuncarg; 229117126Sscottl } else { 230117126Sscottl newtag->dt_lockfunc = dflt_lock; 231117126Sscottl newtag->dt_lockfuncarg = NULL; 232117126Sscottl } 233135030Smarcel 234131223Sscottl newtag->dt_segments = NULL; 235131223Sscottl 236177565Smarius /* Take into account any restrictions imposed by our parent tag. */ 237167308Smarius newtag->dt_lowaddr = ulmin(parent->dt_lowaddr, newtag->dt_lowaddr); 238167308Smarius newtag->dt_highaddr = ulmax(parent->dt_highaddr, newtag->dt_highaddr); 239167308Smarius if (newtag->dt_boundary == 0) 240167308Smarius newtag->dt_boundary = parent->dt_boundary; 241167308Smarius else if (parent->dt_boundary != 0) 242167308Smarius newtag->dt_boundary = ulmin(parent->dt_boundary, 243167308Smarius newtag->dt_boundary); 244167308Smarius atomic_add_int(&parent->dt_ref_count, 1); 24593070Stmm 246135030Smarcel if (newtag->dt_boundary > 0) 247135030Smarcel newtag->dt_maxsegsz = ulmin(newtag->dt_maxsegsz, 248135030Smarcel newtag->dt_boundary); 249135030Smarcel 25086228Stmm *dmat = newtag; 25186228Stmm return (0); 25286228Stmm} 25386228Stmm 25486228Stmmint 25586228Stmmbus_dma_tag_destroy(bus_dma_tag_t dmat) 25686228Stmm{ 25793070Stmm bus_dma_tag_t parent; 25886228Stmm 25986228Stmm if (dmat != NULL) { 260108815Stmm if (dmat->dt_map_count != 0) 26186228Stmm return (EBUSY); 26286228Stmm while (dmat != NULL) { 263108815Stmm parent = dmat->dt_parent; 264112436Smux atomic_subtract_int(&dmat->dt_ref_count, 1); 265108815Stmm if (dmat->dt_ref_count == 0) { 266131223Sscottl if (dmat->dt_segments != NULL) 267131223Sscottl free(dmat->dt_segments, M_DEVBUF); 26886228Stmm free(dmat, M_DEVBUF); 26986228Stmm /* 27086228Stmm * Last reference count, so 27186228Stmm * release our reference 27286228Stmm * count on our parent. 27386228Stmm */ 27486228Stmm dmat = parent; 27586228Stmm } else 27686228Stmm dmat = NULL; 27786228Stmm } 27886228Stmm } 27986228Stmm return (0); 28086228Stmm} 28186228Stmm 282116541Stmm/* Allocate/free a tag, and do the necessary management work. */ 283116541Stmmint 284116541Stmmsparc64_dma_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp) 28586228Stmm{ 28686228Stmm 287131223Sscottl if (dmat->dt_segments == NULL) { 288131223Sscottl dmat->dt_segments = (bus_dma_segment_t *)malloc( 289131223Sscottl sizeof(bus_dma_segment_t) * dmat->dt_nsegments, M_DEVBUF, 290131223Sscottl M_NOWAIT); 291131223Sscottl if (dmat->dt_segments == NULL) 292131223Sscottl return (ENOMEM); 293131223Sscottl } 294104247Sjake *mapp = malloc(sizeof(**mapp), M_DEVBUF, M_NOWAIT | M_ZERO); 295116541Stmm if (*mapp == NULL) 296104247Sjake return (ENOMEM); 297116541Stmm 298116541Stmm SLIST_INIT(&(*mapp)->dm_reslist); 299116541Stmm dmat->dt_map_count++; 300116541Stmm return (0); 30186228Stmm} 30286228Stmm 303116541Stmmvoid 304116541Stmmsparc64_dma_free_map(bus_dma_tag_t dmat, bus_dmamap_t map) 305116541Stmm{ 306116541Stmm 307116541Stmm free(map, M_DEVBUF); 308116541Stmm dmat->dt_map_count--; 309116541Stmm} 310116541Stmm 31186228Stmmstatic int 312116541Stmmnexus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 31386228Stmm{ 31486228Stmm 315116541Stmm return (sparc64_dma_alloc_map(dmat, mapp)); 316116541Stmm} 317116541Stmm 318116541Stmmstatic int 319116541Stmmnexus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 320116541Stmm{ 321116541Stmm 322116541Stmm sparc64_dma_free_map(dmat, map); 32386228Stmm return (0); 32486228Stmm} 32586228Stmm 32686228Stmm/* 327251874Sscottl * Add a single contiguous physical range to the segment list. 328104486Ssam */ 329104486Ssamstatic int 330251874Sscottlnexus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 331251874Sscottl bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 332104486Ssam{ 333251874Sscottl bus_addr_t baddr, bmask; 334104486Ssam int seg; 335104486Ssam 336251874Sscottl /* 337251874Sscottl * Make sure we don't cross any boundaries. 338251874Sscottl */ 339116541Stmm bmask = ~(dmat->dt_boundary - 1); 340251874Sscottl if (dmat->dt_boundary > 0) { 341251874Sscottl baddr = (curaddr + dmat->dt_boundary) & bmask; 342251874Sscottl if (sgsize > (baddr - curaddr)) 343251874Sscottl sgsize = (baddr - curaddr); 344251874Sscottl } 345104486Ssam 346251874Sscottl /* 347251874Sscottl * Insert chunk into a segment, coalescing with 348251874Sscottl * previous segment if possible. 349251874Sscottl */ 350251874Sscottl seg = *segp; 351251874Sscottl if (seg == -1) { 352251874Sscottl seg = 0; 353251874Sscottl segs[seg].ds_addr = curaddr; 354251874Sscottl segs[seg].ds_len = sgsize; 355251874Sscottl } else { 356251874Sscottl if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 357251874Sscottl (segs[seg].ds_len + sgsize) <= dmat->dt_maxsegsz && 358251874Sscottl (dmat->dt_boundary == 0 || 359251874Sscottl (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 360251874Sscottl segs[seg].ds_len += sgsize; 361251874Sscottl else { 362251874Sscottl if (++seg >= dmat->dt_nsegments) 363251874Sscottl return (0); 364104486Ssam segs[seg].ds_addr = curaddr; 365104486Ssam segs[seg].ds_len = sgsize; 366104486Ssam } 367104486Ssam } 368104486Ssam *segp = seg; 369251874Sscottl return (sgsize); 370104486Ssam} 371104486Ssam 372104486Ssam/* 373251874Sscottl * Utility function to load a physical buffer. segp contains 374251874Sscottl * the starting segment on entrace, and the ending segment on exit. 375108821Stmm */ 376108821Stmmstatic int 377251874Sscottlnexus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 378251874Sscottl bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) 379108821Stmm{ 380251874Sscottl bus_addr_t curaddr; 381251874Sscottl bus_size_t sgsize; 382108821Stmm 383251874Sscottl if (segs == NULL) 384251874Sscottl segs = dmat->dt_segments; 385108821Stmm 386251874Sscottl curaddr = buf; 387251874Sscottl while (buflen > 0) { 388251874Sscottl sgsize = MIN(buflen, dmat->dt_maxsegsz); 389251874Sscottl sgsize = nexus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 390251874Sscottl segp); 391251874Sscottl if (sgsize == 0) 392251874Sscottl break; 393251874Sscottl curaddr += sgsize; 394251874Sscottl buflen -= sgsize; 395251874Sscottl } 396108821Stmm 397251874Sscottl /* 398251874Sscottl * Did we fit? 399251874Sscottl */ 400251874Sscottl return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 401108821Stmm} 402108821Stmm 403108821Stmm/* 404251874Sscottl * Utility function to load a linear buffer. segp contains 405251874Sscottl * the starting segment on entrace, and the ending segment on exit. 406104486Ssam */ 407104486Ssamstatic int 408251874Sscottlnexus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 409251874Sscottl bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 410251874Sscottl int *segp) 411104486Ssam{ 412251874Sscottl bus_size_t sgsize; 413251874Sscottl bus_addr_t curaddr; 414251874Sscottl vm_offset_t vaddr = (vm_offset_t)buf; 415104486Ssam 416251874Sscottl if (segs == NULL) 417251874Sscottl segs = dmat->dt_segments; 418104486Ssam 419251874Sscottl while (buflen > 0) { 420251874Sscottl /* 421251874Sscottl * Get the physical address for this segment. 422251874Sscottl */ 423251874Sscottl if (pmap == kernel_pmap) 424251874Sscottl curaddr = pmap_kextract(vaddr); 425251874Sscottl else 426251874Sscottl curaddr = pmap_extract(pmap, vaddr); 427104486Ssam 428251874Sscottl /* 429251874Sscottl * Compute the segment size, and adjust counts. 430251874Sscottl */ 431251874Sscottl sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); 432251874Sscottl if (sgsize > dmat->dt_maxsegsz) 433251874Sscottl sgsize = dmat->dt_maxsegsz; 434251874Sscottl if (buflen < sgsize) 435251874Sscottl sgsize = buflen; 436251874Sscottl 437251874Sscottl sgsize = nexus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 438251874Sscottl segp); 439251874Sscottl if (sgsize == 0) 440251874Sscottl break; 441251874Sscottl 442251874Sscottl vaddr += sgsize; 443251874Sscottl buflen -= sgsize; 444104486Ssam } 445104486Ssam 446251874Sscottl /* 447251874Sscottl * Did we fit? 448251874Sscottl */ 449251874Sscottl return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 450104486Ssam} 451104486Ssam 452251874Sscottlstatic void 453251874Sscottlnexus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 454251874Sscottl struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 455140281Sscottl{ 456140281Sscottl 457140281Sscottl} 458140281Sscottl 459251874Sscottlstatic bus_dma_segment_t * 460251874Sscottlnexus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 461251874Sscottl bus_dma_segment_t *segs, int nsegs, int error) 462104486Ssam{ 463104486Ssam 464251874Sscottl if (segs == NULL) 465251874Sscottl segs = dmat->dt_segments; 466251874Sscottl return (segs); 467104486Ssam} 468104486Ssam 469104486Ssam/* 47086228Stmm * Common function for unloading a DMA map. May be called by 47186228Stmm * bus-specific DMA map unload functions. 47286228Stmm */ 47386228Stmmstatic void 474116541Stmmnexus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 47586228Stmm{ 47686228Stmm 477117390Stmm map->dm_flags &= ~DMF_LOADED; 47886228Stmm} 47986228Stmm 48086228Stmm/* 48186228Stmm * Common function for DMA map synchronization. May be called 48286228Stmm * by bus-specific DMA map synchronization functions. 48386228Stmm */ 48486228Stmmstatic void 485116541Stmmnexus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 48686228Stmm{ 48786228Stmm 48886228Stmm /* 48986228Stmm * We sync out our caches, but the bus must do the same. 49086228Stmm * 49186228Stmm * Actually a #Sync is expensive. We should optimize. 49286228Stmm */ 493113347Smux if ((op & BUS_DMASYNC_PREREAD) || (op & BUS_DMASYNC_PREWRITE)) { 494166096Smarius /* 49586228Stmm * Don't really need to do anything, but flush any pending 496166096Smarius * writes anyway. 49786228Stmm */ 49886228Stmm membar(Sync); 49986228Stmm } 500113347Smux if (op & BUS_DMASYNC_POSTWRITE) { 50186228Stmm /* Nothing to do. Handled by the bus controller. */ 50286228Stmm } 50386228Stmm} 50486228Stmm 50586228Stmm/* 50686228Stmm * Common function for DMA-safe memory allocation. May be called 50786228Stmm * by bus-specific DMA memory allocation functions. 50886228Stmm */ 50986228Stmmstatic int 510116541Stmmnexus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, 511116541Stmm bus_dmamap_t *mapp) 51286228Stmm{ 513118081Smux int mflags; 51493070Stmm 515118081Smux if (flags & BUS_DMA_NOWAIT) 516118081Smux mflags = M_NOWAIT; 517118081Smux else 518118081Smux mflags = M_WAITOK; 519118081Smux if (flags & BUS_DMA_ZERO) 520118081Smux mflags |= M_ZERO; 521118081Smux 522212676Smarius /* 523212676Smarius * XXX: 524212676Smarius * (dmat->dt_alignment < dmat->dt_maxsize) is just a quick hack; the 525212676Smarius * exact alignment guarantees of malloc need to be nailed down, and 526212676Smarius * the code below should be rewritten to take that into account. 527212676Smarius * 528212676Smarius * In the meantime, we'll warn the user if malloc gets it wrong. 529212676Smarius */ 530212676Smarius if (dmat->dt_maxsize <= PAGE_SIZE && 531212676Smarius dmat->dt_alignment < dmat->dt_maxsize) 532118081Smux *vaddr = malloc(dmat->dt_maxsize, M_DEVBUF, mflags); 533212676Smarius else { 53486228Stmm /* 535177565Smarius * XXX use contigmalloc until it is merged into this 536177565Smarius * facility and handles multi-seg allocations. Nobody 537177565Smarius * is doing multi-seg allocations yet though. 53886228Stmm */ 539118081Smux *vaddr = contigmalloc(dmat->dt_maxsize, M_DEVBUF, mflags, 540116541Stmm 0ul, dmat->dt_lowaddr, 541116541Stmm dmat->dt_alignment ? dmat->dt_alignment : 1UL, 542116541Stmm dmat->dt_boundary); 54386228Stmm } 544116541Stmm if (*vaddr == NULL) 54586228Stmm return (ENOMEM); 546213282Sneel if (vtophys(*vaddr) % dmat->dt_alignment) 547212676Smarius printf("%s: failed to align memory properly.\n", __func__); 54886228Stmm return (0); 54986228Stmm} 55086228Stmm 55186228Stmm/* 55286228Stmm * Common function for freeing DMA-safe memory. May be called by 55386228Stmm * bus-specific DMA memory free functions. 55486228Stmm */ 55586228Stmmstatic void 556116541Stmmnexus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 55786228Stmm{ 55886228Stmm 559212676Smarius if (dmat->dt_maxsize <= PAGE_SIZE && 560212676Smarius dmat->dt_alignment < dmat->dt_maxsize) 56186228Stmm free(vaddr, M_DEVBUF); 562212676Smarius else 563116541Stmm contigfree(vaddr, dmat->dt_maxsize, M_DEVBUF); 56486228Stmm} 56586228Stmm 566230687Smariusstatic struct bus_dma_methods nexus_dma_methods = { 567116541Stmm nexus_dmamap_create, 568116541Stmm nexus_dmamap_destroy, 569251874Sscottl nexus_dmamap_load_phys, 570251874Sscottl nexus_dmamap_load_buffer, 571251874Sscottl nexus_dmamap_waitok, 572251874Sscottl nexus_dmamap_complete, 573116541Stmm nexus_dmamap_unload, 574116541Stmm nexus_dmamap_sync, 575116541Stmm nexus_dmamem_alloc, 576116541Stmm nexus_dmamem_free, 577116541Stmm}; 578116541Stmm 57986228Stmmstruct bus_dma_tag nexus_dmatag = { 58086228Stmm NULL, 58186228Stmm NULL, 582167308Smarius 1, 58386228Stmm 0, 584167308Smarius ~0, 585167308Smarius ~0, 58686228Stmm NULL, /* XXX */ 58786228Stmm NULL, 588167308Smarius ~0, 589167308Smarius ~0, 590167308Smarius ~0, 59186228Stmm 0, 59286228Stmm 0, 59386228Stmm 0, 594117126Sscottl NULL, 595117126Sscottl NULL, 596131223Sscottl NULL, 597116541Stmm &nexus_dma_methods, 59886228Stmm}; 59986228Stmm 60086228Stmm/* 60186228Stmm * Helpers to map/unmap bus memory 60286228Stmm */ 60386228Stmmint 604230687Smariusbus_space_map(bus_space_tag_t tag, bus_addr_t address, bus_size_t size, 605230687Smarius int flags, bus_space_handle_t *handlep) 60686228Stmm{ 607230687Smarius 608230687Smarius return (sparc64_bus_mem_map(tag, address, size, flags, 0, handlep)); 609230687Smarius} 610230687Smarius 611230687Smariusint 612230687Smariussparc64_bus_mem_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, 613230687Smarius int flags, vm_offset_t vaddr, bus_space_handle_t *hp) 614230687Smarius{ 61591177Sjake vm_offset_t sva; 61691177Sjake vm_offset_t va; 617113238Sjake vm_paddr_t pa; 61891177Sjake vm_size_t vsz; 61986228Stmm u_long pm_flags; 62086228Stmm 621230687Smarius /* 622230687Smarius * Given that we use physical access for bus_space(9) there's no need 623230687Smarius * need to map anything in unless BUS_SPACE_MAP_LINEAR is requested. 624230687Smarius */ 625230687Smarius if ((flags & BUS_SPACE_MAP_LINEAR) == 0) { 626230687Smarius *hp = addr; 627230687Smarius return (0); 628230687Smarius } 629230687Smarius 630230687Smarius if (tag->bst_cookie == NULL) { 631230687Smarius printf("%s: resource cookie not set\n", __func__); 632230687Smarius return (EINVAL); 633230687Smarius } 634230687Smarius 63586228Stmm size = round_page(size); 63686228Stmm if (size == 0) { 637166096Smarius printf("%s: zero size\n", __func__); 63886228Stmm return (EINVAL); 63986228Stmm } 640230687Smarius 641108815Stmm switch (tag->bst_type) { 64286228Stmm case PCI_CONFIG_BUS_SPACE: 64386228Stmm case PCI_IO_BUS_SPACE: 64486228Stmm case PCI_MEMORY_BUS_SPACE: 64586228Stmm pm_flags = TD_IE; 64686228Stmm break; 64786228Stmm default: 64886228Stmm pm_flags = 0; 64986228Stmm break; 65086228Stmm } 65186228Stmm 652230687Smarius if ((flags & BUS_SPACE_MAP_CACHEABLE) == 0) 65386228Stmm pm_flags |= TD_E; 65486228Stmm 655123865Sobrien if (vaddr != 0L) 65691177Sjake sva = trunc_page(vaddr); 65786228Stmm else { 658123865Sobrien if ((sva = kmem_alloc_nofault(kernel_map, size)) == 0) 659166096Smarius panic("%s: cannot allocate virtual memory", __func__); 66086228Stmm } 66186228Stmm 66286228Stmm pa = trunc_page(addr); 66386228Stmm if ((flags & BUS_SPACE_MAP_READONLY) == 0) 66486228Stmm pm_flags |= TD_W; 66586228Stmm 66691177Sjake va = sva; 66791177Sjake vsz = size; 66886228Stmm do { 66991177Sjake pmap_kenter_flags(va, pa, pm_flags); 67091177Sjake va += PAGE_SIZE; 67186228Stmm pa += PAGE_SIZE; 67291177Sjake } while ((vsz -= PAGE_SIZE) > 0); 67391782Sjake tlb_range_demap(kernel_pmap, sva, sva + size - 1); 674230687Smarius 675230687Smarius /* Note: we preserve the page offset. */ 676230687Smarius rman_set_virtual(tag->bst_cookie, (void *)(sva | (addr & PAGE_MASK))); 67786228Stmm return (0); 67886228Stmm} 67986228Stmm 680230687Smariusvoid 681230687Smariusbus_space_unmap(bus_space_tag_t tag, bus_space_handle_t handle, 682230687Smarius bus_size_t size) 683230687Smarius{ 684230687Smarius 685230687Smarius sparc64_bus_mem_unmap(tag, handle, size); 686230687Smarius} 687230687Smarius 68886228Stmmint 689230687Smariussparc64_bus_mem_unmap(bus_space_tag_t tag, bus_space_handle_t handle, 690230687Smarius bus_size_t size) 69186228Stmm{ 69291177Sjake vm_offset_t sva; 69391177Sjake vm_offset_t va; 69486228Stmm vm_offset_t endva; 69586228Stmm 696230687Smarius if (tag->bst_cookie == NULL || 697230687Smarius (sva = (vm_offset_t)rman_get_virtual(tag->bst_cookie)) == 0) 698230687Smarius return (0); 699230687Smarius sva = trunc_page(sva); 70091177Sjake endva = sva + round_page(size); 70191177Sjake for (va = sva; va < endva; va += PAGE_SIZE) 70298813Sjake pmap_kremove_flags(va); 70391782Sjake tlb_range_demap(kernel_pmap, sva, sva + size - 1); 70493068Stmm kmem_free(kernel_map, sva, size); 70586228Stmm return (0); 70686228Stmm} 70786228Stmm 70886228Stmm/* 709177565Smarius * Fake up a bus tag, for use by console drivers in early boot when the 710177565Smarius * regular means to allocate resources are not yet available. 71186228Stmm * Addr is the physical address of the desired start of the handle. 71286228Stmm */ 71386228Stmmbus_space_handle_t 71486228Stmmsparc64_fake_bustag(int space, bus_addr_t addr, struct bus_space_tag *ptag) 71586228Stmm{ 71686228Stmm 717108815Stmm ptag->bst_cookie = NULL; 718108815Stmm ptag->bst_parent = NULL; 719108815Stmm ptag->bst_type = space; 720119398Smarcel ptag->bst_bus_barrier = nexus_bus_barrier; 72186228Stmm return (addr); 72286228Stmm} 72386228Stmm 72486228Stmm/* 725230687Smarius * Allocate a bus tag. 726230687Smarius */ 727230687Smariusbus_space_tag_t 728230687Smariussparc64_alloc_bus_tag(void *cookie, struct bus_space_tag *ptag, int type, 729230687Smarius void *barrier) 730230687Smarius{ 731230687Smarius bus_space_tag_t bt; 732230687Smarius 733230687Smarius bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF, M_NOWAIT); 734230687Smarius if (bt == NULL) 735230687Smarius return (NULL); 736230687Smarius bt->bst_cookie = cookie; 737230687Smarius bt->bst_parent = ptag; 738230687Smarius bt->bst_type = type; 739230687Smarius bt->bst_bus_barrier = barrier; 740230687Smarius return (bt); 741230687Smarius} 742230687Smarius 743230687Smarius/* 74486228Stmm * Base bus space handlers. 74586228Stmm */ 74686228Stmm 74786228Stmmstatic void 74886228Stmmnexus_bus_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, 74986228Stmm bus_size_t size, int flags) 75086228Stmm{ 75186228Stmm 752166096Smarius /* 75386228Stmm * We have lots of alternatives depending on whether we're 75486228Stmm * synchronizing loads with loads, loads with stores, stores 75586228Stmm * with loads, or stores with stores. The only ones that seem 75686228Stmm * generic are #Sync and #MemIssue. I'll use #Sync for safety. 75786228Stmm */ 75886228Stmm switch(flags) { 75986228Stmm case BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE: 76086228Stmm case BUS_SPACE_BARRIER_READ: 76186228Stmm case BUS_SPACE_BARRIER_WRITE: 76286228Stmm membar(Sync); 76386228Stmm break; 76486228Stmm default: 765166096Smarius panic("%s: unknown flags", __func__); 76686228Stmm } 76786228Stmm return; 76886228Stmm} 76986228Stmm 77086228Stmmstruct bus_space_tag nexus_bustag = { 77186228Stmm NULL, /* cookie */ 77286228Stmm NULL, /* parent bus tag */ 773166096Smarius NEXUS_BUS_SPACE, /* type */ 77486228Stmm nexus_bus_barrier, /* bus_space_barrier */ 77586228Stmm}; 776