ata-dma.c (174576) | ata-dma.c (178067) |
---|---|
1/*- | 1/*- |
2 * Copyright (c) 1998 - 2007 S�ren Schmidt <sos@FreeBSD.org> | 2 * Copyright (c) 1998 - 2008 S�ren Schmidt <sos@FreeBSD.org> |
3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. --- 9 unchanged lines hidden (view full) --- 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> | 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. --- 9 unchanged lines hidden (view full) --- 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> |
28__FBSDID("$FreeBSD: head/sys/dev/ata/ata-dma.c 174576 2007-12-13 11:47:36Z sos $"); | 28__FBSDID("$FreeBSD: head/sys/dev/ata/ata-dma.c 178067 2008-04-10 13:05:05Z sos $"); |
29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/ata.h> 33#include <sys/kernel.h> 34#include <sys/endian.h> 35#include <sys/malloc.h> 36#include <sys/lock.h> 37#include <sys/sema.h> 38#include <sys/taskqueue.h> 39#include <vm/uma.h> 40#include <sys/bus.h> 41#include <machine/bus.h> 42#include <sys/rman.h> 43#include <dev/pci/pcivar.h> 44#include <dev/ata/ata-all.h> 45#include <dev/ata/ata-pci.h> 46 47/* prototypes */ | 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/ata.h> 33#include <sys/kernel.h> 34#include <sys/endian.h> 35#include <sys/malloc.h> 36#include <sys/lock.h> 37#include <sys/sema.h> 38#include <sys/taskqueue.h> 39#include <vm/uma.h> 40#include <sys/bus.h> 41#include <machine/bus.h> 42#include <sys/rman.h> 43#include <dev/pci/pcivar.h> 44#include <dev/ata/ata-all.h> 45#include <dev/ata/ata-pci.h> 46 47/* prototypes */ |
48static void ata_dmaalloc(device_t); 49static void ata_dmafree(device_t); 50static void ata_dmasetprd(void *, bus_dma_segment_t *, int, int); 51static int ata_dmaload(device_t, caddr_t, int32_t, int, void *, int *); 52static int ata_dmaunload(device_t); | 48static void ata_dmaalloc(device_t dev); 49static void ata_dmafree(device_t dev); 50static void ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 51static int ata_dmaload(struct ata_request *request, void *addr, int *nsegs); 52static int ata_dmaunload(struct ata_request *request); |
53 54/* local vars */ 55static MALLOC_DEFINE(M_ATADMA, "ata_dma", "ATA driver DMA"); 56 57/* misc defines */ 58#define MAXTABSZ PAGE_SIZE 59#define MAXWSPCSZ PAGE_SIZE*2 60 61struct ata_dc_cb_args { 62 bus_addr_t maddr; 63 int error; 64}; 65 66void 67ata_dmainit(device_t dev) 68{ 69 struct ata_channel *ch = device_get_softc(dev); 70 | 53 54/* local vars */ 55static MALLOC_DEFINE(M_ATADMA, "ata_dma", "ATA driver DMA"); 56 57/* misc defines */ 58#define MAXTABSZ PAGE_SIZE 59#define MAXWSPCSZ PAGE_SIZE*2 60 61struct ata_dc_cb_args { 62 bus_addr_t maddr; 63 int error; 64}; 65 66void 67ata_dmainit(device_t dev) 68{ 69 struct ata_channel *ch = device_get_softc(dev); 70 |
71 if ((ch->dma = malloc(sizeof(struct ata_dma), M_ATADMA, M_NOWAIT|M_ZERO))) { 72 ch->dma->alloc = ata_dmaalloc; 73 ch->dma->free = ata_dmafree; 74 ch->dma->setprd = ata_dmasetprd; 75 ch->dma->load = ata_dmaload; 76 ch->dma->unload = ata_dmaunload; 77 ch->dma->alignment = 2; 78 ch->dma->boundary = 65536; 79 ch->dma->segsize = 128 * DEV_BSIZE; 80 ch->dma->max_iosize = 128 * DEV_BSIZE; 81 ch->dma->max_address = BUS_SPACE_MAXADDR_32BIT; 82 } | 71 ch->dma.alloc = ata_dmaalloc; 72 ch->dma.free = ata_dmafree; 73 ch->dma.setprd = ata_dmasetprd; 74 ch->dma.load = ata_dmaload; 75 ch->dma.unload = ata_dmaunload; 76 ch->dma.alignment = 2; 77 ch->dma.boundary = 65536; 78 ch->dma.segsize = 63536; 79 ch->dma.max_iosize = 128 * DEV_BSIZE; 80 ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT; |
83} 84 85static void 86ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 87{ | 81} 82 83static void 84ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 85{ |
88 struct ata_dc_cb_args *cba = (struct ata_dc_cb_args *)xsc; | 86 struct ata_dc_cb_args *dcba = (struct ata_dc_cb_args *)xsc; |
89 | 87 |
90 if (!(cba->error = error)) 91 cba->maddr = segs[0].ds_addr; | 88 if (!(dcba->error = error)) 89 dcba->maddr = segs[0].ds_addr; |
92} 93 94static void 95ata_dmaalloc(device_t dev) 96{ 97 struct ata_channel *ch = device_get_softc(dev); | 90} 91 92static void 93ata_dmaalloc(device_t dev) 94{ 95 struct ata_channel *ch = device_get_softc(dev); |
98 struct ata_dc_cb_args ccba; | 96 struct ata_dc_cb_args dcba; |
99 | 97 |
100 if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma->alignment, 0, 101 ch->dma->max_address, BUS_SPACE_MAXADDR, 102 NULL, NULL, ch->dma->max_iosize, 103 ATA_DMA_ENTRIES, ch->dma->segsize, 104 0, NULL, NULL, &ch->dma->dmatag)) | 98 if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0, 99 ch->dma.max_address, BUS_SPACE_MAXADDR, 100 NULL, NULL, ch->dma.max_iosize, 101 ATA_DMA_ENTRIES, ch->dma.segsize, 102 0, NULL, NULL, &ch->dma.dmatag)) |
105 goto error; 106 | 103 goto error; 104 |
107 if (bus_dma_tag_create(ch->dma->dmatag, PAGE_SIZE, PAGE_SIZE, 108 ch->dma->max_address, BUS_SPACE_MAXADDR, 109 NULL, NULL, MAXTABSZ, 1, MAXTABSZ, 110 0, NULL, NULL, &ch->dma->sg_tag)) 111 goto error; 112 113 if (bus_dma_tag_create(ch->dma->dmatag,ch->dma->alignment,ch->dma->boundary, 114 ch->dma->max_address, BUS_SPACE_MAXADDR, 115 NULL, NULL, ch->dma->max_iosize, 116 ATA_DMA_ENTRIES, ch->dma->segsize, 117 0, NULL, NULL, &ch->dma->data_tag)) 118 goto error; 119 120 if (bus_dmamem_alloc(ch->dma->sg_tag, (void **)&ch->dma->sg, 0, 121 &ch->dma->sg_map)) 122 goto error; 123 124 if (bus_dmamap_load(ch->dma->sg_tag, ch->dma->sg_map, ch->dma->sg, 125 MAXTABSZ, ata_dmasetupc_cb, &ccba, 0) || ccba.error) { 126 bus_dmamem_free(ch->dma->sg_tag, ch->dma->sg, ch->dma->sg_map); 127 goto error; 128 } 129 ch->dma->sg_bus = ccba.maddr; 130 131 if (bus_dmamap_create(ch->dma->data_tag, 0, &ch->dma->data_map)) 132 goto error; 133 134 if (bus_dma_tag_create(ch->dma->dmatag, PAGE_SIZE, 64 * 1024, 135 ch->dma->max_address, BUS_SPACE_MAXADDR, | 105 if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, 64 * 1024, 106 ch->dma.max_address, BUS_SPACE_MAXADDR, |
136 NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ, | 107 NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ, |
137 0, NULL, NULL, &ch->dma->work_tag)) | 108 0, NULL, NULL, &ch->dma.work_tag)) |
138 goto error; 139 | 109 goto error; 110 |
140 if (bus_dmamem_alloc(ch->dma->work_tag, (void **)&ch->dma->work, 0, 141 &ch->dma->work_map)) | 111 if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0, 112 &ch->dma.work_map)) |
142 goto error; 143 | 113 goto error; 114 |
144 if (bus_dmamap_load(ch->dma->work_tag, ch->dma->work_map,ch->dma->work, 145 MAXWSPCSZ, ata_dmasetupc_cb, &ccba, 0) || ccba.error) { 146 bus_dmamem_free(ch->dma->work_tag,ch->dma->work, ch->dma->work_map); | 115 if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map ,ch->dma.work, 116 MAXWSPCSZ, ata_dmasetupc_cb, &dcba, BUS_DMA_NOWAIT) || 117 dcba.error) { 118 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); |
147 goto error; 148 } | 119 goto error; 120 } |
149 ch->dma->work_bus = ccba.maddr; | 121 ch->dma.work_bus = dcba.maddr; |
150 151 return; 152 153error: 154 device_printf(dev, "WARNING - DMA allocation failed, disabling DMA\n"); 155 ata_dmafree(dev); | 122 123 return; 124 125error: 126 device_printf(dev, "WARNING - DMA allocation failed, disabling DMA\n"); 127 ata_dmafree(dev); |
156 free(ch->dma, M_ATADMA); 157 ch->dma = NULL; | |
158} 159 160static void 161ata_dmafree(device_t dev) 162{ 163 struct ata_channel *ch = device_get_softc(dev); 164 | 128} 129 130static void 131ata_dmafree(device_t dev) 132{ 133 struct ata_channel *ch = device_get_softc(dev); 134 |
165 if (ch->dma->work_bus) { 166 bus_dmamap_unload(ch->dma->work_tag, ch->dma->work_map); 167 bus_dmamem_free(ch->dma->work_tag, ch->dma->work, ch->dma->work_map); 168 ch->dma->work_bus = 0; 169 ch->dma->work_map = NULL; 170 ch->dma->work = NULL; | 135 if (ch->dma.work_bus) { 136 bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); 137 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 138 ch->dma.work_bus = 0; 139 ch->dma.work_map = NULL; 140 ch->dma.work = NULL; |
171 } | 141 } |
172 if (ch->dma->work_tag) { 173 bus_dma_tag_destroy(ch->dma->work_tag); 174 ch->dma->work_tag = NULL; | 142 if (ch->dma.work_tag) { 143 bus_dma_tag_destroy(ch->dma.work_tag); 144 ch->dma.work_tag = NULL; |
175 } | 145 } |
176 if (ch->dma->sg_bus) { 177 bus_dmamap_unload(ch->dma->sg_tag, ch->dma->sg_map); 178 bus_dmamem_free(ch->dma->sg_tag, ch->dma->sg, ch->dma->sg_map); 179 ch->dma->sg_bus = 0; 180 ch->dma->sg_map = NULL; 181 ch->dma->sg = NULL; | 146 if (ch->dma.dmatag) { 147 bus_dma_tag_destroy(ch->dma.dmatag); 148 ch->dma.dmatag = NULL; |
182 } | 149 } |
183 if (ch->dma->data_map) { 184 bus_dmamap_destroy(ch->dma->data_tag, ch->dma->data_map); 185 ch->dma->data_map = NULL; 186 } 187 if (ch->dma->sg_tag) { 188 bus_dma_tag_destroy(ch->dma->sg_tag); 189 ch->dma->sg_tag = NULL; 190 } 191 if (ch->dma->data_tag) { 192 bus_dma_tag_destroy(ch->dma->data_tag); 193 ch->dma->data_tag = NULL; 194 } 195 if (ch->dma->dmatag) { 196 bus_dma_tag_destroy(ch->dma->dmatag); 197 ch->dma->dmatag = NULL; 198 } | |
199} 200 201static void 202ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 203{ 204 struct ata_dmasetprd_args *args = xsc; 205 struct ata_dma_prdentry *prd = args->dmatab; 206 int i; --- 6 unchanged lines hidden (view full) --- 213 prd[i].count = htole32(segs[i].ds_len); 214 } 215 prd[i - 1].count |= htole32(ATA_DMA_EOT); 216 KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); 217 args->nsegs = nsegs; 218} 219 220static int | 150} 151 152static void 153ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 154{ 155 struct ata_dmasetprd_args *args = xsc; 156 struct ata_dma_prdentry *prd = args->dmatab; 157 int i; --- 6 unchanged lines hidden (view full) --- 164 prd[i].count = htole32(segs[i].ds_len); 165 } 166 prd[i - 1].count |= htole32(ATA_DMA_EOT); 167 KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); 168 args->nsegs = nsegs; 169} 170 171static int |
221ata_dmaload(device_t dev, caddr_t data, int32_t count, int dir, 222 void *addr, int *entries) | 172ata_dmaload(struct ata_request *request, void *addr, int *entries) |
223{ | 173{ |
224 struct ata_channel *ch = device_get_softc(dev); 225 struct ata_dmasetprd_args cba; | 174 struct ata_channel *ch = device_get_softc(request->parent); 175 struct ata_dc_cb_args dcba; 176 struct ata_dmasetprd_args dspa; |
226 int error; 227 | 177 int error; 178 |
228 if (ch->dma->flags & ATA_DMA_LOADED) { 229 device_printf(dev, "FAILURE - already active DMA on this device\n"); | 179 ATA_DEBUG_RQ(request, "dmaload"); 180 181 if (request->dma.cur_iosize) { 182 device_printf(request->dev, 183 "FAILURE - already active DMA on this device\n"); |
230 return EIO; 231 } | 184 return EIO; 185 } |
232 if (!count) { 233 device_printf(dev, "FAILURE - zero length DMA transfer attempted\n"); | 186 if (!request->bytecount) { 187 device_printf(request->dev, 188 "FAILURE - zero length DMA transfer attempted\n"); |
234 return EIO; 235 } | 189 return EIO; 190 } |
236 if (((uintptr_t)data & (ch->dma->alignment - 1)) || 237 (count & (ch->dma->alignment - 1))) { 238 device_printf(dev, "FAILURE - non aligned DMA transfer attempted\n"); | 191 if (((uintptr_t)(request->data) & (ch->dma.alignment - 1)) || 192 (request->bytecount & (ch->dma.alignment - 1))) { 193 device_printf(request->dev, 194 "FAILURE - non aligned DMA transfer attempted\n"); |
239 return EIO; 240 } | 195 return EIO; 196 } |
241 if (count > ch->dma->max_iosize) { 242 device_printf(dev, "FAILURE - oversized DMA transfer attempt %d > %d\n", 243 count, ch->dma->max_iosize); | 197 if (request->bytecount > ch->dma.max_iosize) { 198 device_printf(request->dev, 199 "FAILURE - oversized DMA transfer attempt %d > %d\n", 200 request->bytecount, ch->dma.max_iosize); |
244 return EIO; 245 } 246 | 201 return EIO; 202 } 203 |
247 cba.dmatab = addr; | 204 if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE, 205 ch->dma.max_address, BUS_SPACE_MAXADDR, 206 NULL, NULL, MAXTABSZ, 1, MAXTABSZ, 207 0, NULL, NULL, &request->dma.sg_tag)) { 208 device_printf(request->dev, "FAILURE - create sg_tag\n"); 209 goto error; 210 } |
248 | 211 |
249 if ((error = bus_dmamap_load(ch->dma->data_tag, ch->dma->data_map, 250 data, count, ch->dma->setprd, &cba, 251 BUS_DMA_NOWAIT)) || (error = cba.error)) 252 return error; | 212 if (bus_dmamem_alloc(request->dma.sg_tag, (void **)&request->dma.sg, 0, 213 &request->dma.sg_map)) { 214 device_printf(request->dev, "FAILURE - alloc sg_map\n"); 215 goto error; 216 } |
253 | 217 |
254 *entries = cba.nsegs; | 218 if (bus_dmamap_load(request->dma.sg_tag, request->dma.sg_map, 219 request->dma.sg, MAXTABSZ, 220 ata_dmasetupc_cb, &dcba, BUS_DMA_NOWAIT) || dcba.error){ 221 bus_dmamem_free(request->dma.sg_tag, 222 request->dma.sg, request->dma.sg_map); 223 device_printf(request->dev, "FAILURE - load sg\n"); 224 goto error; 225 } 226 request->dma.sg_bus = dcba.maddr; |
255 | 227 |
256 bus_dmamap_sync(ch->dma->sg_tag, ch->dma->sg_map, BUS_DMASYNC_PREWRITE); | 228 if (bus_dma_tag_create(ch->dma.dmatag, ch->dma.alignment, ch->dma.boundary, 229 ch->dma.max_address, BUS_SPACE_MAXADDR, 230 NULL, NULL, ch->dma.max_iosize, 231 ATA_DMA_ENTRIES, ch->dma.segsize, 232 BUS_DMA_ALLOCNOW, NULL, NULL, 233 &request->dma.data_tag)) { 234 device_printf(request->dev, "FAILURE - create data_tag\n"); 235 goto error; 236 } |
257 | 237 |
258 bus_dmamap_sync(ch->dma->data_tag, ch->dma->data_map, 259 dir ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); | 238 if (bus_dmamap_create(request->dma.data_tag, 0, &request->dma.data_map)) { 239 device_printf(request->dev, "FAILURE - create data_map\n"); 240 goto error; 241 } |
260 | 242 |
261 ch->dma->cur_iosize = count; 262 ch->dma->flags = dir ? (ATA_DMA_LOADED | ATA_DMA_READ) : ATA_DMA_LOADED; | 243 if (addr) 244 dspa.dmatab = addr; 245 else 246 dspa.dmatab = request->dma.sg; 247 248 if ((error = bus_dmamap_load(request->dma.data_tag, request->dma.data_map, 249 request->data, request->bytecount, 250 ch->dma.setprd, &dspa, BUS_DMA_NOWAIT)) || 251 (error = dspa.error)) { 252 device_printf(request->dev, "FAILURE - load data\n"); 253 goto error; 254 } 255 256 if (entries) 257 *entries = dspa.nsegs; 258 259 bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map, 260 BUS_DMASYNC_PREWRITE); 261 262 bus_dmamap_sync(request->dma.data_tag, request->dma.data_map, 263 (request->flags & ATA_R_READ) ? 264 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 265 266 request->dma.cur_iosize = request->bytecount; 267 |
263 return 0; | 268 return 0; |
269 270error: 271 ata_dmaunload(request); 272 return EIO; |
|
264} 265 266int | 273} 274 275int |
267ata_dmaunload(device_t dev) | 276ata_dmaunload(struct ata_request *request) |
268{ | 277{ |
269 struct ata_channel *ch = device_get_softc(dev); | 278 ATA_DEBUG_RQ(request, "dmaunload"); |
270 | 279 |
271 if (ch->dma->flags & ATA_DMA_LOADED) { 272 bus_dmamap_sync(ch->dma->sg_tag, ch->dma->sg_map, | 280 if (request->dma.cur_iosize) { 281 bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map, |
273 BUS_DMASYNC_POSTWRITE); 274 | 282 BUS_DMASYNC_POSTWRITE); 283 |
275 bus_dmamap_sync(ch->dma->data_tag, ch->dma->data_map, 276 (ch->dma->flags & ATA_DMA_READ) ? | 284 bus_dmamap_sync(request->dma.data_tag, request->dma.data_map, 285 (request->flags & ATA_R_READ) ? |
277 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); | 286 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); |
278 bus_dmamap_unload(ch->dma->data_tag, ch->dma->data_map); | 287 bus_dmamap_unload(request->dma.data_tag, request->dma.data_map); |
279 | 288 |
280 ch->dma->cur_iosize = 0; 281 ch->dma->flags &= ~ATA_DMA_LOADED; | 289 request->dma.cur_iosize = 0; |
282 } | 290 } |
291 if (request->dma.data_map) { 292 bus_dmamap_destroy(request->dma.data_tag, request->dma.data_map); 293 request->dma.data_map = NULL; 294 } 295 if (request->dma.data_tag) { 296 bus_dma_tag_destroy(request->dma.data_tag); 297 request->dma.data_tag = NULL; 298 } 299 if (request->dma.sg_bus) { 300 bus_dmamap_unload(request->dma.sg_tag, request->dma.sg_map); 301 bus_dmamem_free(request->dma.sg_tag, request->dma.sg, 302 request->dma.sg_map); 303 request->dma.sg = NULL; 304 request->dma.sg_bus = 0; 305 request->dma.sg_map = NULL; 306 } 307 if (request->dma.sg_tag) { 308 bus_dma_tag_destroy(request->dma.sg_tag); 309 request->dma.sg_tag = NULL; 310 } |
|
283 return 0; 284} | 311 return 0; 312} |