ata-dma.c revision 173768
145095Ssos/*- 2166878Ssos * Copyright (c) 1998 - 2007 S�ren Schmidt <sos@FreeBSD.org> 345095Ssos * All rights reserved. 445095Ssos * 545095Ssos * Redistribution and use in source and binary forms, with or without 645095Ssos * modification, are permitted provided that the following conditions 745095Ssos * are met: 845095Ssos * 1. Redistributions of source code must retain the above copyright 945095Ssos * notice, this list of conditions and the following disclaimer, 1045095Ssos * without modification, immediately at the beginning of the file. 1145095Ssos * 2. Redistributions in binary form must reproduce the above copyright 1245095Ssos * notice, this list of conditions and the following disclaimer in the 1345095Ssos * documentation and/or other materials provided with the distribution. 1445095Ssos * 1545095Ssos * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1645095Ssos * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1745095Ssos * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1845095Ssos * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1945095Ssos * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2045095Ssos * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2145095Ssos * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2245095Ssos * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2345095Ssos * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2445095Ssos * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2545095Ssos */ 2645095Ssos 27119418Sobrien#include <sys/cdefs.h> 28119418Sobrien__FBSDID("$FreeBSD: head/sys/dev/ata/ata-dma.c 173768 2007-11-20 04:52:19Z kevlo $"); 29119418Sobrien 3045095Ssos#include <sys/param.h> 3145095Ssos#include <sys/systm.h> 3274302Ssos#include <sys/ata.h> 33111188Ssos#include <sys/kernel.h> 3495533Smike#include <sys/endian.h> 3545095Ssos#include <sys/malloc.h> 36117126Sscottl#include <sys/lock.h> 37124403Ssos#include <sys/sema.h> 38119404Ssos#include <sys/taskqueue.h> 39124534Ssos#include <vm/uma.h> 4045798Ssos#include <sys/bus.h> 4166106Ssos#include <machine/bus.h> 4272106Ssos#include <sys/rman.h> 43119404Ssos#include <dev/pci/pcivar.h> 4445095Ssos#include <dev/ata/ata-all.h> 45111188Ssos#include <dev/ata/ata-pci.h> 4645095Ssos 4752067Ssos/* prototypes */ 48145713Ssosstatic void ata_dmaalloc(device_t); 49145713Ssosstatic void ata_dmafree(device_t); 50133637Ssosstatic void ata_dmasetprd(void *, bus_dma_segment_t *, int, int); 51145818Ssosstatic int ata_dmaload(device_t, caddr_t, int32_t, int, void *, int *); 52145713Ssosstatic int ata_dmaunload(device_t); 5352067Ssos 54111188Ssos/* local vars */ 55151897Srwatsonstatic MALLOC_DEFINE(M_ATADMA, "ata_dma", "ATA driver DMA"); 56111188Ssos 5752067Ssos/* misc defines */ 58144330Ssos#define MAXTABSZ PAGE_SIZE 59145713Ssos#define MAXWSPCSZ PAGE_SIZE*2 6045720Speter 6193882Ssosstruct ata_dc_cb_args { 6293882Ssos bus_addr_t maddr; 6393882Ssos int error; 6493882Ssos}; 6593882Ssos 66119453Ssosvoid 67145713Ssosata_dmainit(device_t dev) 68111188Ssos{ 69145713Ssos struct ata_channel *ch = device_get_softc(dev); 70145713Ssos 71119453Ssos if ((ch->dma = malloc(sizeof(struct ata_dma), M_ATADMA, M_NOWAIT|M_ZERO))) { 72119453Ssos ch->dma->alloc = ata_dmaalloc; 73119453Ssos ch->dma->free = ata_dmafree; 74133637Ssos ch->dma->setprd = ata_dmasetprd; 75121310Ssos ch->dma->load = ata_dmaload; 76121310Ssos ch->dma->unload = ata_dmaunload; 77119453Ssos ch->dma->alignment = 2; 78151002Ssos ch->dma->boundary = 128 * DEV_BSIZE; 79151002Ssos ch->dma->segsize = 128 * DEV_BSIZE; 80135034Ssos ch->dma->max_iosize = 128 * DEV_BSIZE; 81168500Ssos ch->dma->max_address = BUS_SPACE_MAXADDR_32BIT; 82119453Ssos } 83111188Ssos} 84111188Ssos 8593882Ssosstatic void 8693882Ssosata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 8766070Ssos{ 8893882Ssos struct ata_dc_cb_args *cba = (struct ata_dc_cb_args *)xsc; 8966070Ssos 9093882Ssos if (!(cba->error = error)) 9193882Ssos cba->maddr = segs[0].ds_addr; 9293882Ssos} 9393882Ssos 94119453Ssosstatic void 95145713Ssosata_dmaalloc(device_t dev) 9693882Ssos{ 97145713Ssos struct ata_channel *ch = device_get_softc(dev); 9893882Ssos struct ata_dc_cb_args ccba; 9993882Ssos 100161928Sjmg if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma->alignment, 0, 101168500Ssos ch->dma->max_address, BUS_SPACE_MAXADDR, 102151002Ssos NULL, NULL, ch->dma->max_iosize, 103151002Ssos ATA_DMA_ENTRIES, ch->dma->segsize, 104137809Sscottl 0, NULL, NULL, &ch->dma->dmatag)) 105119453Ssos goto error; 106119453Ssos 107120883Ssos if (bus_dma_tag_create(ch->dma->dmatag, PAGE_SIZE, PAGE_SIZE, 108168500Ssos ch->dma->max_address, BUS_SPACE_MAXADDR, 109120883Ssos NULL, NULL, MAXTABSZ, 1, MAXTABSZ, 110144330Ssos 0, NULL, NULL, &ch->dma->sg_tag)) 111119453Ssos goto error; 112119453Ssos 113123034Ssos if (bus_dma_tag_create(ch->dma->dmatag,ch->dma->alignment,ch->dma->boundary, 114168500Ssos ch->dma->max_address, BUS_SPACE_MAXADDR, 115151002Ssos NULL, NULL, ch->dma->max_iosize, 116151002Ssos ATA_DMA_ENTRIES, ch->dma->segsize, 117153142Ssos 0, NULL, NULL, &ch->dma->data_tag)) 118119453Ssos goto error; 119119453Ssos 120144330Ssos if (bus_dmamem_alloc(ch->dma->sg_tag, (void **)&ch->dma->sg, 0, 121144330Ssos &ch->dma->sg_map)) 122119453Ssos goto error; 123119453Ssos 124144330Ssos if (bus_dmamap_load(ch->dma->sg_tag, ch->dma->sg_map, ch->dma->sg, 125119453Ssos MAXTABSZ, ata_dmasetupc_cb, &ccba, 0) || ccba.error) { 126144330Ssos bus_dmamem_free(ch->dma->sg_tag, ch->dma->sg, ch->dma->sg_map); 127119453Ssos goto error; 128112791Ssos } 129144330Ssos ch->dma->sg_bus = ccba.maddr; 130128183Ssos 131144330Ssos if (bus_dmamap_create(ch->dma->data_tag, 0, &ch->dma->data_map)) 132119453Ssos goto error; 133128183Ssos 134145713Ssos if (bus_dma_tag_create(ch->dma->dmatag, PAGE_SIZE, 64 * 1024, 135168500Ssos ch->dma->max_address, BUS_SPACE_MAXADDR, 136128183Ssos NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ, 137144330Ssos 0, NULL, NULL, &ch->dma->work_tag)) 138128183Ssos goto error; 139128183Ssos 140144330Ssos if (bus_dmamem_alloc(ch->dma->work_tag, (void **)&ch->dma->work, 0, 141144330Ssos &ch->dma->work_map)) 142128183Ssos goto error; 143128183Ssos 144144330Ssos if (bus_dmamap_load(ch->dma->work_tag, ch->dma->work_map,ch->dma->work, 145128183Ssos MAXWSPCSZ, ata_dmasetupc_cb, &ccba, 0) || ccba.error) { 146144330Ssos bus_dmamem_free(ch->dma->work_tag,ch->dma->work, ch->dma->work_map); 147128183Ssos goto error; 148128183Ssos } 149144330Ssos ch->dma->work_bus = ccba.maddr; 150128183Ssos 151119453Ssos return; 15293882Ssos 153119453Ssoserror: 154145713Ssos device_printf(dev, "WARNING - DMA allocation failed, disabling DMA\n"); 155145713Ssos ata_dmafree(dev); 156119453Ssos free(ch->dma, M_ATADMA); 157119453Ssos ch->dma = NULL; 15866070Ssos} 15966070Ssos 160111188Ssosstatic void 161145713Ssosata_dmafree(device_t dev) 16245095Ssos{ 163145713Ssos struct ata_channel *ch = device_get_softc(dev); 164145713Ssos 165144330Ssos if (ch->dma->work_bus) { 166144330Ssos bus_dmamap_unload(ch->dma->work_tag, ch->dma->work_map); 167144330Ssos bus_dmamem_free(ch->dma->work_tag, ch->dma->work, ch->dma->work_map); 168144330Ssos ch->dma->work_bus = 0; 169144330Ssos ch->dma->work_map = NULL; 170144330Ssos ch->dma->work = NULL; 171128183Ssos } 172144330Ssos if (ch->dma->work_tag) { 173144330Ssos bus_dma_tag_destroy(ch->dma->work_tag); 174144330Ssos ch->dma->work_tag = NULL; 175128183Ssos } 176144330Ssos if (ch->dma->sg_bus) { 177144330Ssos bus_dmamap_unload(ch->dma->sg_tag, ch->dma->sg_map); 178144330Ssos bus_dmamem_free(ch->dma->sg_tag, ch->dma->sg, ch->dma->sg_map); 179144330Ssos ch->dma->sg_bus = 0; 180144330Ssos ch->dma->sg_map = NULL; 181144330Ssos ch->dma->sg = NULL; 18293882Ssos } 183144330Ssos if (ch->dma->data_map) { 184144330Ssos bus_dmamap_destroy(ch->dma->data_tag, ch->dma->data_map); 185144330Ssos ch->dma->data_map = NULL; 18693882Ssos } 187144330Ssos if (ch->dma->sg_tag) { 188144330Ssos bus_dma_tag_destroy(ch->dma->sg_tag); 189144330Ssos ch->dma->sg_tag = NULL; 19093882Ssos } 191144330Ssos if (ch->dma->data_tag) { 192144330Ssos bus_dma_tag_destroy(ch->dma->data_tag); 193144330Ssos ch->dma->data_tag = NULL; 19493882Ssos } 195112791Ssos if (ch->dma->dmatag) { 196112791Ssos bus_dma_tag_destroy(ch->dma->dmatag); 197112791Ssos ch->dma->dmatag = NULL; 19893882Ssos } 19993882Ssos} 20093882Ssos 20193882Ssosstatic void 202133637Ssosata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 20393882Ssos{ 204133637Ssos struct ata_dmasetprd_args *args = xsc; 205133637Ssos struct ata_dma_prdentry *prd = args->dmatab; 206133637Ssos int i; 20793882Ssos 208133637Ssos if ((args->error = error)) 20993882Ssos return; 210133637Ssos 21193882Ssos for (i = 0; i < nsegs; i++) { 212133637Ssos prd[i].addr = htole32(segs[i].ds_addr); 213133637Ssos prd[i].count = htole32(segs[i].ds_len); 21493882Ssos } 215133637Ssos prd[i - 1].count |= htole32(ATA_DMA_EOT); 216173768Skevlo KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); 217145818Ssos args->nsegs = nsegs; 21893882Ssos} 21993882Ssos 220111188Ssosstatic int 221145818Ssosata_dmaload(device_t dev, caddr_t data, int32_t count, int dir, 222145818Ssos void *addr, int *entries) 22345095Ssos{ 224145769Ssos struct ata_channel *ch = device_get_softc(dev); 225133637Ssos struct ata_dmasetprd_args cba; 226153142Ssos int error; 22745095Ssos 228133637Ssos if (ch->dma->flags & ATA_DMA_LOADED) { 229145713Ssos device_printf(dev, "FAILURE - already active DMA on this device\n"); 230153142Ssos return EIO; 23166070Ssos } 23245095Ssos if (!count) { 233145713Ssos device_printf(dev, "FAILURE - zero length DMA transfer attempted\n"); 234153142Ssos return EIO; 23545095Ssos } 236121310Ssos if (((uintptr_t)data & (ch->dma->alignment - 1)) || 237121310Ssos (count & (ch->dma->alignment - 1))) { 238145713Ssos device_printf(dev, "FAILURE - non aligned DMA transfer attempted\n"); 239153142Ssos return EIO; 240121310Ssos } 241119404Ssos if (count > ch->dma->max_iosize) { 242145713Ssos device_printf(dev, "FAILURE - oversized DMA transfer attempt %d > %d\n", 243144330Ssos count, ch->dma->max_iosize); 244153142Ssos return EIO; 245119404Ssos } 24645095Ssos 247145818Ssos cba.dmatab = addr; 24895010Ssos 249153142Ssos if ((error = bus_dmamap_load(ch->dma->data_tag, ch->dma->data_map, 250153142Ssos data, count, ch->dma->setprd, &cba, 251153142Ssos BUS_DMA_NOWAIT)) || (error = cba.error)) 252153142Ssos return error; 25393882Ssos 254145818Ssos *entries = cba.nsegs; 255145818Ssos 256144800Ssos bus_dmamap_sync(ch->dma->sg_tag, ch->dma->sg_map, BUS_DMASYNC_PREWRITE); 257144800Ssos 258144330Ssos bus_dmamap_sync(ch->dma->data_tag, ch->dma->data_map, 259112791Ssos dir ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 26095010Ssos 261121310Ssos ch->dma->cur_iosize = count; 262168503Ssos ch->dma->flags = dir ? (ATA_DMA_LOADED | ATA_DMA_READ) : ATA_DMA_LOADED; 26395010Ssos return 0; 26445095Ssos} 26545095Ssos 26666070Ssosint 267145713Ssosata_dmaunload(device_t dev) 26845095Ssos{ 269145769Ssos struct ata_channel *ch = device_get_softc(dev); 270119404Ssos 271145818Ssos if (ch->dma->flags & ATA_DMA_LOADED) { 272145818Ssos bus_dmamap_sync(ch->dma->sg_tag, ch->dma->sg_map, 273145818Ssos BUS_DMASYNC_POSTWRITE); 274119404Ssos 275145818Ssos bus_dmamap_sync(ch->dma->data_tag, ch->dma->data_map, 276168493Ssos (ch->dma->flags & ATA_DMA_READ) ? 277145818Ssos BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 278145818Ssos bus_dmamap_unload(ch->dma->data_tag, ch->dma->data_map); 279145818Ssos 280145818Ssos ch->dma->cur_iosize = 0; 281145818Ssos ch->dma->flags &= ~ATA_DMA_LOADED; 282145818Ssos } 283112791Ssos return 0; 28445095Ssos} 285