subr_bus_dma.c revision 248893
1133819Stjr/*- 2133819Stjr * Copyright (c) 2012 EMC Corp. 3133819Stjr * All rights reserved. 4133819Stjr * 5133819Stjr * Copyright (c) 1997, 1998 Justin T. Gibbs. 6158407Snetchild * All rights reserved. 7133819Stjr * 8133819Stjr * Redistribution and use in source and binary forms, with or without 9133819Stjr * modification, are permitted provided that the following conditions 10133819Stjr * are met: 11133819Stjr * 1. Redistributions of source code must retain the above copyright 12133819Stjr * notice, this list of conditions and the following disclaimer. 13133819Stjr * 2. Redistributions in binary form must reproduce the above copyright 14133819Stjr * notice, this list of conditions and the following disclaimer in the 15133819Stjr * documentation and/or other materials provided with the distribution. 16133819Stjr * 17133819Stjr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18133819Stjr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19133819Stjr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20133819Stjr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21133819Stjr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22133819Stjr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23133819Stjr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24133819Stjr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25133819Stjr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26133819Stjr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27133819Stjr * SUCH DAMAGE. 28133819Stjr */ 29133819Stjr 30133819Stjr#include <sys/cdefs.h> 31133819Stjr__FBSDID("$FreeBSD: head/sys/kern/subr_bus_dma.c 248893 2013-03-29 16:00:16Z jimharris $"); 32133819Stjr 33133819Stjr#include "opt_bus.h" 34133819Stjr 35143198Ssobomax#include <sys/param.h> 36133819Stjr#include <sys/conf.h> 37133819Stjr#include <sys/systm.h> 38133819Stjr#include <sys/bio.h> 39133819Stjr#include <sys/bus.h> 40133819Stjr#include <sys/callout.h> 41133819Stjr#include <sys/mbuf.h> 42133819Stjr#include <sys/memdesc.h> 43133819Stjr#include <sys/proc.h> 44133819Stjr#include <sys/uio.h> 45133819Stjr 46133819Stjr#include <vm/vm.h> 47133819Stjr#include <vm/vm_page.h> 48133819Stjr#include <vm/vm_map.h> 49133819Stjr#include <vm/pmap.h> 50133819Stjr 51133819Stjr#include <cam/cam.h> 52133819Stjr#include <cam/cam_ccb.h> 53133819Stjr 54133819Stjr#include <machine/bus.h> 55133819Stjr 56133819Stjr/* 57133819Stjr * Load a list of virtual addresses. 58133819Stjr */ 59133819Stjrstatic int 60133819Stjr_bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map, 61133819Stjr bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs, 62133819Stjr int flags) 63133819Stjr{ 64133819Stjr int error; 65133819Stjr 66133819Stjr error = 0; 67133819Stjr for (; sglist_cnt > 0; sglist_cnt--, list++) { 68133819Stjr error = _bus_dmamap_load_buffer(dmat, map, 69133819Stjr (void *)list->ds_addr, list->ds_len, pmap, flags, NULL, 70133819Stjr nsegs); 71133819Stjr if (error) 72133819Stjr break; 73133819Stjr } 74133819Stjr return (error); 75133819Stjr} 76133819Stjr 77133819Stjr/* 78133819Stjr * Load a list of physical addresses. 79133819Stjr */ 80133819Stjrstatic int 81133819Stjr_bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map, 82133819Stjr bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags) 83133819Stjr{ 84133819Stjr int error; 85156919Snetchild 86156919Snetchild error = 0; 87156919Snetchild for (; sglist_cnt > 0; sglist_cnt--, list++) { 88156919Snetchild error = _bus_dmamap_load_phys(dmat, map, 89133819Stjr (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL, 90133819Stjr nsegs); 91133819Stjr if (error) 92133819Stjr break; 93133819Stjr } 94133819Stjr return (error); 95133819Stjr} 96133819Stjr 97133819Stjr/* 98133819Stjr * Load an mbuf chain. 99133819Stjr */ 100133819Stjrstatic int 101133819Stjr_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 102133819Stjr struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) 103133819Stjr{ 104133819Stjr struct mbuf *m; 105133819Stjr int error; 106133819Stjr 107133819Stjr M_ASSERTPKTHDR(m0); 108133819Stjr 109133819Stjr error = 0; 110133819Stjr for (m = m0; m != NULL && error == 0; m = m->m_next) { 111133819Stjr if (m->m_len > 0) { 112133819Stjr error = _bus_dmamap_load_buffer(dmat, map, m->m_data, 113133819Stjr m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF, 114133819Stjr segs, nsegs); 115133819Stjr } 116133819Stjr } 117133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 118133819Stjr __func__, dmat, flags, error, *nsegs); 119133819Stjr return (error); 120133819Stjr} 121133819Stjr 122133819Stjr/* 123133819Stjr * Load from block io. 124133819Stjr */ 125133819Stjrstatic int 126133819Stjr_bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, 127133819Stjr int *nsegs, int flags) 128133819Stjr{ 129133819Stjr vm_paddr_t paddr; 130133819Stjr bus_size_t len, tlen; 131133819Stjr int error, i, ma_offs; 132133819Stjr 133133819Stjr if ((bio->bio_flags & BIO_UNMAPPED) == 0) { 134133819Stjr error = _bus_dmamap_load_buffer(dmat, map, bio->bio_data, 135133819Stjr bio->bio_bcount, kernel_pmap, flags, NULL, nsegs); 136133819Stjr return (error); 137133819Stjr } 138133819Stjr 139133819Stjr error = 0; 140133819Stjr tlen = bio->bio_bcount; 141133819Stjr ma_offs = bio->bio_ma_offset; 142133819Stjr for (i = 0; tlen > 0; i++, tlen -= len) { 143133819Stjr len = min(PAGE_SIZE - ma_offs, tlen); 144133819Stjr paddr = VM_PAGE_TO_PHYS(bio->bio_ma[i]) + ma_offs; 145133819Stjr error = _bus_dmamap_load_phys(dmat, map, paddr, len, 146133819Stjr flags, NULL, nsegs); 147133819Stjr if (error != 0) 148133819Stjr break; 149133819Stjr ma_offs = 0; 150133819Stjr } 151133819Stjr return (error); 152133819Stjr} 153133819Stjr 154133819Stjr/* 155133819Stjr * Load a cam control block. 156133819Stjr */ 157133819Stjrstatic int 158133819Stjr_bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 159133819Stjr int *nsegs, int flags) 160133819Stjr{ 161133819Stjr struct ccb_ataio *ataio; 162133819Stjr struct ccb_scsiio *csio; 163133819Stjr struct ccb_hdr *ccb_h; 164133819Stjr void *data_ptr; 165133819Stjr int error; 166133819Stjr uint32_t dxfer_len; 167133819Stjr uint16_t sglist_cnt; 168133819Stjr 169133819Stjr error = 0; 170133819Stjr ccb_h = &ccb->ccb_h; 171133819Stjr switch (ccb_h->func_code) { 172133819Stjr case XPT_SCSI_IO: 173133819Stjr csio = &ccb->csio; 174133819Stjr data_ptr = csio->data_ptr; 175133819Stjr dxfer_len = csio->dxfer_len; 176133819Stjr sglist_cnt = csio->sglist_cnt; 177133819Stjr break; 178133819Stjr case XPT_ATA_IO: 179133819Stjr ataio = &ccb->ataio; 180133819Stjr data_ptr = ataio->data_ptr; 181133819Stjr dxfer_len = ataio->dxfer_len; 182133819Stjr sglist_cnt = 0; 183133819Stjr break; 184133819Stjr default: 185133819Stjr panic("_bus_dmamap_load_ccb: Unsupported func code %d", 186133819Stjr ccb_h->func_code); 187133819Stjr } 188133819Stjr 189133819Stjr switch ((ccb_h->flags & CAM_DATA_MASK)) { 190133819Stjr case CAM_DATA_VADDR: 191133819Stjr error = _bus_dmamap_load_buffer(dmat, map, data_ptr, dxfer_len, 192133819Stjr kernel_pmap, flags, NULL, nsegs); 193133819Stjr break; 194133819Stjr case CAM_DATA_PADDR: 195133819Stjr error = _bus_dmamap_load_phys(dmat, map, 196133819Stjr (vm_paddr_t)(uintptr_t)data_ptr, dxfer_len, flags, NULL, 197133819Stjr nsegs); 198133819Stjr break; 199133819Stjr case CAM_DATA_SG: 200133819Stjr error = _bus_dmamap_load_vlist(dmat, map, 201133819Stjr (bus_dma_segment_t *)data_ptr, sglist_cnt, kernel_pmap, 202133819Stjr nsegs, flags); 203133819Stjr break; 204133819Stjr case CAM_DATA_SG_PADDR: 205133819Stjr error = _bus_dmamap_load_plist(dmat, map, 206133819Stjr (bus_dma_segment_t *)data_ptr, sglist_cnt, nsegs, flags); 207133819Stjr break; 208133819Stjr case CAM_DATA_BIO: 209133819Stjr error = _bus_dmamap_load_bio(dmat, map, (struct bio *)data_ptr, 210133819Stjr nsegs, flags); 211133819Stjr break; 212133819Stjr default: 213133819Stjr panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented", 214133819Stjr ccb_h->flags); 215133819Stjr } 216133819Stjr return (error); 217133819Stjr} 218133819Stjr 219133819Stjr/* 220133819Stjr * Load a uio. 221133819Stjr */ 222133819Stjrstatic int 223133819Stjr_bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 224133819Stjr int *nsegs, int flags) 225133819Stjr{ 226133819Stjr bus_size_t resid; 227156843Snetchild bus_size_t minlen; 228156843Snetchild struct iovec *iov; 229156843Snetchild pmap_t pmap; 230156843Snetchild caddr_t addr; 231133819Stjr int error, i; 232133819Stjr 233133819Stjr if (uio->uio_segflg == UIO_USERSPACE) { 234133819Stjr KASSERT(uio->uio_td != NULL, 235133819Stjr ("bus_dmamap_load_uio: USERSPACE but no proc")); 236133819Stjr pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 237133819Stjr } else 238133819Stjr pmap = kernel_pmap; 239133819Stjr resid = uio->uio_resid; 240133819Stjr iov = uio->uio_iov; 241133819Stjr error = 0; 242133819Stjr 243133819Stjr for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 244133819Stjr /* 245133819Stjr * Now at the first iovec to load. Load each iovec 246133819Stjr * until we have exhausted the residual count. 247133819Stjr */ 248133819Stjr 249133819Stjr addr = (caddr_t) iov[i].iov_base; 250133819Stjr minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 251133819Stjr if (minlen > 0) { 252133819Stjr error = _bus_dmamap_load_buffer(dmat, map, addr, 253133819Stjr minlen, pmap, flags, NULL, nsegs); 254133819Stjr resid -= minlen; 255133819Stjr } 256133819Stjr } 257133819Stjr 258133819Stjr return (error); 259133819Stjr} 260133819Stjr 261133819Stjr/* 262133819Stjr * Map the buffer buf into bus space using the dmamap map. 263133819Stjr */ 264133819Stjrint 265133819Stjrbus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 266156919Snetchild bus_size_t buflen, bus_dmamap_callback_t *callback, 267156843Snetchild void *callback_arg, int flags) 268156919Snetchild{ 269156843Snetchild bus_dma_segment_t *segs; 270133819Stjr struct memdesc mem; 271133819Stjr int error; 272133819Stjr int nsegs; 273133819Stjr 274133819Stjr if ((flags & BUS_DMA_NOWAIT) == 0) { 275133819Stjr mem = memdesc_vaddr(buf, buflen); 276133819Stjr _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 277133819Stjr } 278133819Stjr 279133819Stjr nsegs = -1; 280133819Stjr error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap, 281133819Stjr flags, NULL, &nsegs); 282133819Stjr nsegs++; 283133819Stjr 284133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 285133819Stjr __func__, dmat, flags, error, nsegs); 286133819Stjr 287133819Stjr if (error == EINPROGRESS) 288133819Stjr return (error); 289133819Stjr 290133819Stjr segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 291133819Stjr if (error) 292133819Stjr (*callback)(callback_arg, segs, 0, error); 293156843Snetchild else 294156843Snetchild (*callback)(callback_arg, segs, nsegs, 0); 295156843Snetchild 296156843Snetchild /* 297147142Ssobomax * Return ENOMEM to the caller so that it can pass it up the stack. 298147142Ssobomax * This error only happens when NOWAIT is set, so deferral is disabled. 299147142Ssobomax */ 300147142Ssobomax if (error == ENOMEM) 301133819Stjr return (error); 302133819Stjr 303133819Stjr return (0); 304133819Stjr} 305133819Stjr 306133819Stjrint 307133819Stjrbus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 308133819Stjr bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 309133819Stjr{ 310133819Stjr bus_dma_segment_t *segs; 311133819Stjr int nsegs, error; 312133819Stjr 313133819Stjr flags |= BUS_DMA_NOWAIT; 314133819Stjr nsegs = -1; 315133819Stjr error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); 316133819Stjr ++nsegs; 317133819Stjr 318133819Stjr segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 319133819Stjr if (error) 320133819Stjr (*callback)(callback_arg, segs, 0, 0, error); 321133819Stjr else 322133819Stjr (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); 323133819Stjr 324133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 325133819Stjr __func__, dmat, flags, error, nsegs); 326133819Stjr return (error); 327133819Stjr} 328133819Stjr 329133819Stjrint 330133819Stjrbus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 331133819Stjr bus_dma_segment_t *segs, int *nsegs, int flags) 332133819Stjr{ 333133819Stjr int error; 334133819Stjr 335133819Stjr flags |= BUS_DMA_NOWAIT; 336133819Stjr *nsegs = -1; 337133819Stjr error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); 338133819Stjr ++*nsegs; 339133819Stjr _bus_dmamap_complete(dmat, map, segs, *nsegs, error); 340133819Stjr return (error); 341133819Stjr} 342133819Stjr 343133819Stjrint 344133819Stjrbus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 345133819Stjr bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 346133819Stjr{ 347133819Stjr bus_dma_segment_t *segs; 348133819Stjr int nsegs, error; 349133819Stjr 350133819Stjr flags |= BUS_DMA_NOWAIT; 351133819Stjr nsegs = -1; 352133819Stjr error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); 353133819Stjr nsegs++; 354133819Stjr 355133819Stjr segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 356133819Stjr if (error) 357133819Stjr (*callback)(callback_arg, segs, 0, 0, error); 358133819Stjr else 359133819Stjr (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); 360133819Stjr 361133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 362133819Stjr __func__, dmat, flags, error, nsegs); 363133819Stjr return (error); 364133819Stjr} 365133819Stjr 366133819Stjrint 367133819Stjrbus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 368133819Stjr bus_dmamap_callback_t *callback, void *callback_arg, 369133819Stjr int flags) 370133819Stjr{ 371133819Stjr bus_dma_segment_t *segs; 372133819Stjr struct ccb_hdr *ccb_h; 373133819Stjr struct memdesc mem; 374133819Stjr int error; 375133819Stjr int nsegs; 376133819Stjr 377133819Stjr ccb_h = &ccb->ccb_h; 378133819Stjr if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { 379133819Stjr callback(callback_arg, NULL, 0, 0); 380133819Stjr return (0); 381133819Stjr } 382133819Stjr if ((flags & BUS_DMA_NOWAIT) == 0) { 383133819Stjr mem = memdesc_ccb(ccb); 384133819Stjr _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 385133819Stjr } 386133819Stjr nsegs = -1; 387133819Stjr error = _bus_dmamap_load_ccb(dmat, map, ccb, &nsegs, flags); 388133819Stjr nsegs++; 389133819Stjr 390133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 391133819Stjr __func__, dmat, flags, error, nsegs); 392133819Stjr 393133819Stjr if (error == EINPROGRESS) 394133819Stjr return (error); 395133819Stjr 396133819Stjr segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 397133819Stjr if (error) 398133819Stjr (*callback)(callback_arg, segs, 0, error); 399133819Stjr else 400133819Stjr (*callback)(callback_arg, segs, nsegs, error); 401133819Stjr /* 402133819Stjr * Return ENOMEM to the caller so that it can pass it up the stack. 403133819Stjr * This error only happens when NOWAIT is set, so deferral is disabled. 404133819Stjr */ 405133819Stjr if (error == ENOMEM) 406133819Stjr return (error); 407133819Stjr 408133819Stjr return (0); 409133819Stjr} 410133819Stjr 411133819Stjrint 412133819Stjrbus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map, 413133819Stjr struct memdesc *mem, bus_dmamap_callback_t *callback, 414133819Stjr void *callback_arg, int flags) 415133819Stjr{ 416133819Stjr bus_dma_segment_t *segs; 417133819Stjr int error; 418133819Stjr int nsegs; 419133819Stjr 420133819Stjr if ((flags & BUS_DMA_NOWAIT) == 0) 421133819Stjr _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); 422133819Stjr 423133819Stjr nsegs = -1; 424133819Stjr error = 0; 425133819Stjr switch (mem->md_type) { 426133819Stjr case MEMDESC_VADDR: 427133819Stjr error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr, 428133819Stjr mem->md_opaque, kernel_pmap, flags, NULL, &nsegs); 429133819Stjr break; 430133819Stjr case MEMDESC_PADDR: 431133819Stjr error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr, 432133819Stjr mem->md_opaque, flags, NULL, &nsegs); 433133819Stjr break; 434133819Stjr case MEMDESC_VLIST: 435133819Stjr error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list, 436133819Stjr mem->md_opaque, kernel_pmap, &nsegs, flags); 437133819Stjr break; 438133819Stjr case MEMDESC_PLIST: 439133819Stjr error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list, 440133819Stjr mem->md_opaque, &nsegs, flags); 441133819Stjr break; 442133819Stjr case MEMDESC_BIO: 443133819Stjr error = _bus_dmamap_load_bio(dmat, map, mem->u.md_bio, 444133819Stjr &nsegs, flags); 445133819Stjr break; 446133819Stjr case MEMDESC_UIO: 447133819Stjr error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio, 448133819Stjr &nsegs, flags); 449133819Stjr break; 450133819Stjr case MEMDESC_MBUF: 451133819Stjr error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf, 452133819Stjr NULL, &nsegs, flags); 453133819Stjr break; 454133819Stjr case MEMDESC_CCB: 455133819Stjr error = _bus_dmamap_load_ccb(dmat, map, mem->u.md_ccb, &nsegs, 456133819Stjr flags); 457133819Stjr break; 458133819Stjr } 459133819Stjr nsegs++; 460133819Stjr 461133819Stjr CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 462133819Stjr __func__, dmat, flags, error, nsegs); 463133819Stjr 464133819Stjr if (error == EINPROGRESS) 465133819Stjr return (error); 466133819Stjr 467133819Stjr segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 468133819Stjr if (error) 469133819Stjr (*callback)(callback_arg, segs, 0, error); 470133819Stjr else 471133819Stjr (*callback)(callback_arg, segs, nsegs, 0); 472133819Stjr 473133819Stjr /* 474133819Stjr * Return ENOMEM to the caller so that it can pass it up the stack. 475133819Stjr * This error only happens when NOWAIT is set, so deferral is disabled. 476133819Stjr */ 477133819Stjr if (error == ENOMEM) 478133819Stjr return (error); 479133819Stjr 480133819Stjr return (0); 481133819Stjr} 482133819Stjr