subr_bus_dma.c revision 259510
174462Salfred/*- 274462Salfred * Copyright (c) 2012 EMC Corp. 374462Salfred * All rights reserved. 474462Salfred * 574462Salfred * Copyright (c) 1997, 1998 Justin T. Gibbs. 674462Salfred * All rights reserved. 774462Salfred * 874462Salfred * Redistribution and use in source and binary forms, with or without 974462Salfred * modification, are permitted provided that the following conditions 1074462Salfred * are met: 1174462Salfred * 1. Redistributions of source code must retain the above copyright 1274462Salfred * notice, this list of conditions and the following disclaimer. 1374462Salfred * 2. Redistributions in binary form must reproduce the above copyright 1474462Salfred * notice, this list of conditions and the following disclaimer in the 1574462Salfred * documentation and/or other materials provided with the distribution. 1674462Salfred * 1774462Salfred * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1874462Salfred * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1974462Salfred * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2074462Salfred * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2174462Salfred * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2274462Salfred * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2374462Salfred * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2474462Salfred * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2574462Salfred * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2674462Salfred * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2774462Salfred * SUCH DAMAGE. 2874462Salfred */ 2974462Salfred 3074462Salfred#include <sys/cdefs.h> 3174462Salfred__FBSDID("$FreeBSD: stable/10/sys/kern/subr_bus_dma.c 259510 2013-12-17 13:38:21Z kib $"); 3274462Salfred 3374462Salfred#include "opt_bus.h" 3474462Salfred 3574462Salfred#include <sys/param.h> 3674462Salfred#include <sys/conf.h> 3774462Salfred#include <sys/systm.h> 3874462Salfred#include <sys/bio.h> 3974462Salfred#include <sys/bus.h> 4074462Salfred#include <sys/callout.h> 4174462Salfred#include <sys/mbuf.h> 4274462Salfred#include <sys/memdesc.h> 4374462Salfred#include <sys/proc.h> 4474462Salfred#include <sys/uio.h> 4574462Salfred 4674462Salfred#include <vm/vm.h> 4774462Salfred#include <vm/vm_page.h> 4874462Salfred#include <vm/vm_map.h> 4974462Salfred#include <vm/pmap.h> 5074462Salfred 51136320Sstefanf#include <cam/cam.h> 5274462Salfred#include <cam/cam_ccb.h> 5374462Salfred 5474462Salfred#include <machine/bus.h> 5574462Salfred 5674462Salfred/* 5774462Salfred * Load a list of virtual addresses. 5874462Salfred */ 5974462Salfredstatic int 6074462Salfred_bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map, 6174462Salfred bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs, 6274462Salfred int flags) 6374462Salfred{ 6474462Salfred int error; 65141217Skuriyama 66141217Skuriyama error = 0; 6792969Salfred for (; sglist_cnt > 0; sglist_cnt--, list++) { 6892909Salfred error = _bus_dmamap_load_buffer(dmat, map, 6992909Salfred (void *)(uintptr_t)list->ds_addr, list->ds_len, pmap, 7074462Salfred flags, NULL, nsegs); 7174462Salfred if (error) 7274462Salfred break; 7374462Salfred } 7474462Salfred return (error); 7574462Salfred} 7674462Salfred 7774462Salfred/* 7874462Salfred * Load a list of physical addresses. 79260250Sdelphij */ 8074462Salfredstatic int 8174462Salfred_bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map, 8274462Salfred bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags) 8374462Salfred{ 8474462Salfred int error; 8574462Salfred 8674462Salfred error = 0; 8774462Salfred for (; sglist_cnt > 0; sglist_cnt--, list++) { 8874462Salfred error = _bus_dmamap_load_phys(dmat, map, 8974462Salfred (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL, 9074462Salfred nsegs); 9174462Salfred if (error) 9284923Salfred break; 9384923Salfred } 9484923Salfred return (error); 9584923Salfred} 9684923Salfred 9784923Salfred/* 9884923Salfred * Load an mbuf chain. 9984923Salfred */ 100260250Sdelphijstatic int 10184923Salfred_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 10284923Salfred struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) 10384923Salfred{ 10486319Salfred struct mbuf *m; 10584923Salfred int error; 10684923Salfred 10784923Salfred error = 0; 10884923Salfred for (m = m0; m != NULL && error == 0; m = m->m_next) { 10984923Salfred if (m->m_len > 0) { 11084923Salfred error = _bus_dmamap_load_buffer(dmat, map, m->m_data, 11184923Salfred m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF, 11284923Salfred segs, nsegs); 11384923Salfred } 11484923Salfred } 115299986Struckman CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 11684923Salfred __func__, dmat, flags, error, *nsegs); 11784923Salfred return (error); 11884923Salfred} 11984923Salfred 12084923Salfred/* 12184923Salfred * Load from block io. 12284923Salfred */ 12384923Salfredstatic int 12484923Salfred_bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, 12574462Salfred int *nsegs, int flags) 12674462Salfred{ 12774462Salfred int error; 12874462Salfred 12974462Salfred if ((bio->bio_flags & BIO_UNMAPPED) == 0) { 13074462Salfred error = _bus_dmamap_load_buffer(dmat, map, bio->bio_data, 13174462Salfred bio->bio_bcount, kernel_pmap, flags, NULL, nsegs); 13274462Salfred } else { 13374462Salfred error = _bus_dmamap_load_ma(dmat, map, bio->bio_ma, 13474462Salfred bio->bio_bcount, bio->bio_ma_offset, flags, NULL, nsegs); 13574462Salfred } 13674462Salfred return (error); 13774462Salfred} 13874462Salfred 13974462Salfredint 14074462Salfredbus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map, 14174462Salfred struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 14274462Salfred bus_dma_segment_t *segs, int *segp) 14374462Salfred{ 14474462Salfred vm_paddr_t paddr; 14574462Salfred bus_size_t len; 14674462Salfred int error, i; 14774462Salfred 14874462Salfred error = 0; 14974462Salfred for (i = 0; tlen > 0; i++, tlen -= len) { 15074462Salfred len = min(PAGE_SIZE - ma_offs, tlen); 15174462Salfred paddr = VM_PAGE_TO_PHYS(ma[i]) + ma_offs; 15274462Salfred error = _bus_dmamap_load_phys(dmat, map, paddr, len, 15374462Salfred flags, segs, segp); 15474462Salfred if (error != 0) 15574462Salfred break; 15674462Salfred ma_offs = 0; 15776093Salfred } 15874462Salfred return (error); 15974462Salfred} 16074462Salfred 161260250Sdelphij/* 16274462Salfred * Load a cam control block. 16374462Salfred */ 16474462Salfredstatic int 16574462Salfred_bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 16674462Salfred int *nsegs, int flags) 16774462Salfred{ 16874462Salfred struct ccb_hdr *ccb_h; 16974462Salfred void *data_ptr; 17074462Salfred int error; 17174462Salfred uint32_t dxfer_len; 17274462Salfred uint16_t sglist_cnt; 17374462Salfred 17474462Salfred error = 0; 17574462Salfred ccb_h = &ccb->ccb_h; 17674462Salfred switch (ccb_h->func_code) { 17774462Salfred case XPT_SCSI_IO: { 17874462Salfred struct ccb_scsiio *csio; 17974462Salfred 18074462Salfred csio = &ccb->csio; 18174462Salfred data_ptr = csio->data_ptr; 18274462Salfred dxfer_len = csio->dxfer_len; 18374462Salfred sglist_cnt = csio->sglist_cnt; 18474462Salfred break; 18574462Salfred } 18674462Salfred case XPT_CONT_TARGET_IO: { 18774462Salfred struct ccb_scsiio *ctio; 188260250Sdelphij 18974462Salfred ctio = &ccb->ctio; 19074462Salfred data_ptr = ctio->data_ptr; 19174462Salfred dxfer_len = ctio->dxfer_len; 192141217Skuriyama sglist_cnt = ctio->sglist_cnt; 19392977Salfred break; 19474462Salfred } 19574462Salfred case XPT_ATA_IO: { 196126606Sroam struct ccb_ataio *ataio; 197126606Sroam 19874462Salfred ataio = &ccb->ataio; 19974462Salfred data_ptr = ataio->data_ptr; 20074462Salfred dxfer_len = ataio->dxfer_len; 20174462Salfred sglist_cnt = 0; 20274462Salfred break; 20374462Salfred } 20474462Salfred default: 20574462Salfred panic("_bus_dmamap_load_ccb: Unsupported func code %d", 20674462Salfred ccb_h->func_code); 20774462Salfred } 20874462Salfred 20974462Salfred switch ((ccb_h->flags & CAM_DATA_MASK)) { 21074462Salfred case CAM_DATA_VADDR: 21174462Salfred error = _bus_dmamap_load_buffer(dmat, map, data_ptr, dxfer_len, 21274462Salfred kernel_pmap, flags, NULL, nsegs); 21374462Salfred break; 21474462Salfred case CAM_DATA_PADDR: 21574462Salfred error = _bus_dmamap_load_phys(dmat, map, 21674462Salfred (vm_paddr_t)(uintptr_t)data_ptr, dxfer_len, flags, NULL, 21774462Salfred nsegs); 21876093Salfred break; 21974462Salfred case CAM_DATA_SG: 22074462Salfred error = _bus_dmamap_load_vlist(dmat, map, 22174462Salfred (bus_dma_segment_t *)data_ptr, sglist_cnt, kernel_pmap, 22274462Salfred nsegs, flags); 22374462Salfred break; 22474462Salfred case CAM_DATA_SG_PADDR: 22574462Salfred error = _bus_dmamap_load_plist(dmat, map, 22676093Salfred (bus_dma_segment_t *)data_ptr, sglist_cnt, nsegs, flags); 22776093Salfred break; 22876093Salfred case CAM_DATA_BIO: 22974462Salfred error = _bus_dmamap_load_bio(dmat, map, (struct bio *)data_ptr, 23074462Salfred nsegs, flags); 23174462Salfred break; 23274462Salfred default: 23374462Salfred panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented", 23474462Salfred ccb_h->flags); 23574462Salfred } 23674462Salfred return (error); 23774462Salfred} 23874462Salfred 239141217Skuriyama/* 240141217Skuriyama * Load a uio. 241141217Skuriyama */ 242141217Skuriyamastatic int 243141217Skuriyama_bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 24474462Salfred int *nsegs, int flags) 24574462Salfred{ 24674462Salfred bus_size_t resid; 24774462Salfred bus_size_t minlen; 24874462Salfred struct iovec *iov; 24974462Salfred pmap_t pmap; 25074462Salfred caddr_t addr; 25174462Salfred int error, i; 25274462Salfred 25374462Salfred if (uio->uio_segflg == UIO_USERSPACE) { 25474462Salfred KASSERT(uio->uio_td != NULL, 25574462Salfred ("bus_dmamap_load_uio: USERSPACE but no proc")); 25674462Salfred pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 25774462Salfred } else 25874462Salfred pmap = kernel_pmap; 25974462Salfred resid = uio->uio_resid; 26074462Salfred iov = uio->uio_iov; 26174462Salfred error = 0; 26274462Salfred 26374462Salfred for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 26474462Salfred /* 26574462Salfred * Now at the first iovec to load. Load each iovec 26674462Salfred * until we have exhausted the residual count. 26774462Salfred */ 26874462Salfred 26974462Salfred addr = (caddr_t) iov[i].iov_base; 27074462Salfred minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 27174462Salfred if (minlen > 0) { 27274462Salfred error = _bus_dmamap_load_buffer(dmat, map, addr, 27374462Salfred minlen, pmap, flags, NULL, nsegs); 274126606Sroam resid -= minlen; 275126606Sroam } 276126606Sroam } 277126606Sroam 278126606Sroam return (error); 279126606Sroam} 280126606Sroam 281126606Sroam/* 282126606Sroam * Map the buffer buf into bus space using the dmamap map. 283126606Sroam */ 284126606Sroamint 285126606Sroambus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 286126606Sroam bus_size_t buflen, bus_dmamap_callback_t *callback, 287126606Sroam void *callback_arg, int flags) 288126606Sroam{ 289126606Sroam bus_dma_segment_t *segs; 29074462Salfred struct memdesc mem; 29174462Salfred int error; 29274462Salfred int nsegs; 29374462Salfred 29476093Salfred if ((flags & BUS_DMA_NOWAIT) == 0) { 29574462Salfred mem = memdesc_vaddr(buf, buflen); 296132254Smr _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 29774462Salfred } 29874462Salfred 29974462Salfred nsegs = -1; 30074462Salfred error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap, 30174462Salfred flags, NULL, &nsegs); 30274462Salfred nsegs++; 30374462Salfred 30474462Salfred CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 30574462Salfred __func__, dmat, flags, error, nsegs); 30674462Salfred 30774462Salfred if (error == EINPROGRESS) 30874462Salfred return (error); 30974462Salfred 31074462Salfred segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 31174462Salfred if (error) 31274462Salfred (*callback)(callback_arg, segs, 0, error); 31374462Salfred else 31474462Salfred (*callback)(callback_arg, segs, nsegs, 0); 31574462Salfred 31674462Salfred /* 31774462Salfred * Return ENOMEM to the caller so that it can pass it up the stack. 31874462Salfred * This error only happens when NOWAIT is set, so deferral is disabled. 31974462Salfred */ 32074462Salfred if (error == ENOMEM) 32174462Salfred return (error); 32274462Salfred 323260250Sdelphij return (0); 32474462Salfred} 32574462Salfred 32674462Salfredint 32774462Salfredbus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 32874462Salfred bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 32974462Salfred{ 33074462Salfred bus_dma_segment_t *segs; 33174462Salfred int nsegs, error; 33274462Salfred 33374462Salfred M_ASSERTPKTHDR(m0); 334121558Speter 335121558Speter flags |= BUS_DMA_NOWAIT; 33674462Salfred nsegs = -1; 33774462Salfred error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); 33874462Salfred ++nsegs; 33974462Salfred 34074462Salfred segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 34174462Salfred if (error) 34274462Salfred (*callback)(callback_arg, segs, 0, 0, error); 34374462Salfred else 34474462Salfred (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); 34574462Salfred 34674462Salfred CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 34774462Salfred __func__, dmat, flags, error, nsegs); 34874462Salfred return (error); 34974462Salfred} 35074462Salfred 351260250Sdelphijint 35274462Salfredbus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 35374462Salfred bus_dma_segment_t *segs, int *nsegs, int flags) 35474462Salfred{ 35574462Salfred int error; 35674462Salfred 35774462Salfred flags |= BUS_DMA_NOWAIT; 35874462Salfred *nsegs = -1; 35974462Salfred error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); 36074462Salfred ++*nsegs; 36174462Salfred _bus_dmamap_complete(dmat, map, segs, *nsegs, error); 362121558Speter return (error); 363121558Speter} 364121558Speter 36574462Salfredint 36674462Salfredbus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 36774462Salfred bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 36874462Salfred{ 36974462Salfred bus_dma_segment_t *segs; 37074462Salfred int nsegs, error; 37174462Salfred 37274462Salfred flags |= BUS_DMA_NOWAIT; 37374462Salfred nsegs = -1; 37474462Salfred error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); 37574462Salfred nsegs++; 376260250Sdelphij 37774462Salfred segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 378132254Smr if (error) 379132254Smr (*callback)(callback_arg, segs, 0, 0, error); 380132254Smr else 381132254Smr (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); 38274462Salfred 38374462Salfred CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 38474462Salfred __func__, dmat, flags, error, nsegs); 38574462Salfred return (error); 38674462Salfred} 38774462Salfred 38874462Salfredint 38974462Salfredbus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 39074462Salfred bus_dmamap_callback_t *callback, void *callback_arg, 39174462Salfred int flags) 39274462Salfred{ 39374462Salfred bus_dma_segment_t *segs; 39474462Salfred struct ccb_hdr *ccb_h; 39574462Salfred struct memdesc mem; 39674462Salfred int error; 39774462Salfred int nsegs; 39874462Salfred 39974462Salfred ccb_h = &ccb->ccb_h; 40074462Salfred if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { 40174462Salfred callback(callback_arg, NULL, 0, 0); 40274462Salfred return (0); 40374462Salfred } 40474462Salfred if ((flags & BUS_DMA_NOWAIT) == 0) { 40574462Salfred mem = memdesc_ccb(ccb); 40674462Salfred _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 40774462Salfred } 40874462Salfred nsegs = -1; 40974462Salfred error = _bus_dmamap_load_ccb(dmat, map, ccb, &nsegs, flags); 41074462Salfred nsegs++; 41174462Salfred 41274462Salfred CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 41374462Salfred __func__, dmat, flags, error, nsegs); 41474462Salfred 41574462Salfred if (error == EINPROGRESS) 41674462Salfred return (error); 41774462Salfred 41874462Salfred segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 419260250Sdelphij if (error) 42074462Salfred (*callback)(callback_arg, segs, 0, error); 42174462Salfred else 42274462Salfred (*callback)(callback_arg, segs, nsegs, error); 42374462Salfred /* 42474462Salfred * Return ENOMEM to the caller so that it can pass it up the stack. 42574462Salfred * This error only happens when NOWAIT is set, so deferral is disabled. 42674462Salfred */ 42774462Salfred if (error == ENOMEM) 42874462Salfred return (error); 42984923Salfred 43074462Salfred return (0); 43174462Salfred} 43274462Salfred 43374462Salfredint 43474462Salfredbus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, 43574462Salfred bus_dmamap_callback_t *callback, void *callback_arg, 43674462Salfred int flags) 43774462Salfred{ 43874462Salfred bus_dma_segment_t *segs; 43974462Salfred struct memdesc mem; 44074462Salfred int error; 44174462Salfred int nsegs; 44274462Salfred 44374462Salfred if ((flags & BUS_DMA_NOWAIT) == 0) { 44474462Salfred mem = memdesc_bio(bio); 44574462Salfred _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 44674462Salfred } 44774462Salfred nsegs = -1; 44874462Salfred error = _bus_dmamap_load_bio(dmat, map, bio, &nsegs, flags); 44974462Salfred nsegs++; 45074462Salfred 451260250Sdelphij CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 45274462Salfred __func__, dmat, flags, error, nsegs); 45374462Salfred 45474462Salfred if (error == EINPROGRESS) 45574462Salfred return (error); 45674462Salfred 45774462Salfred segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 45874462Salfred if (error) 45974462Salfred (*callback)(callback_arg, segs, 0, error); 46074462Salfred else 46174462Salfred (*callback)(callback_arg, segs, nsegs, error); 46274462Salfred /* 46374462Salfred * Return ENOMEM to the caller so that it can pass it up the stack. 46474462Salfred * This error only happens when NOWAIT is set, so deferral is disabled. 46574462Salfred */ 46674462Salfred if (error == ENOMEM) 46784923Salfred return (error); 46874462Salfred 46974462Salfred return (0); 47074462Salfred} 47174462Salfred 47274462Salfredint 47374462Salfredbus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map, 47474462Salfred struct memdesc *mem, bus_dmamap_callback_t *callback, 47574462Salfred void *callback_arg, int flags) 47674462Salfred{ 47774462Salfred bus_dma_segment_t *segs; 47874462Salfred int error; 47974462Salfred int nsegs; 48074462Salfred 48174462Salfred if ((flags & BUS_DMA_NOWAIT) == 0) 48274462Salfred _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); 48374462Salfred 48474462Salfred nsegs = -1; 48574462Salfred error = 0; 48674462Salfred switch (mem->md_type) { 48774462Salfred case MEMDESC_VADDR: 48874462Salfred error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr, 489121558Speter mem->md_opaque, kernel_pmap, flags, NULL, &nsegs); 490121558Speter break; 491121558Speter case MEMDESC_PADDR: 49274462Salfred error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr, 49374462Salfred mem->md_opaque, flags, NULL, &nsegs); 49474462Salfred break; 49574462Salfred case MEMDESC_VLIST: 49674462Salfred error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list, 49774462Salfred mem->md_opaque, kernel_pmap, &nsegs, flags); 49874462Salfred break; 49974462Salfred case MEMDESC_PLIST: 50074462Salfred error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list, 50174462Salfred mem->md_opaque, &nsegs, flags); 50274462Salfred break; 50374462Salfred case MEMDESC_BIO: 50474462Salfred error = _bus_dmamap_load_bio(dmat, map, mem->u.md_bio, 50574462Salfred &nsegs, flags); 506260250Sdelphij break; 50774462Salfred case MEMDESC_UIO: 50874462Salfred error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio, 50974462Salfred &nsegs, flags); 51074462Salfred break; 51174462Salfred case MEMDESC_MBUF: 51274462Salfred error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf, 51374462Salfred NULL, &nsegs, flags); 51474462Salfred break; 51574462Salfred case MEMDESC_CCB: 51674462Salfred error = _bus_dmamap_load_ccb(dmat, map, mem->u.md_ccb, &nsegs, 51774462Salfred flags); 51874462Salfred break; 51974462Salfred } 52074462Salfred nsegs++; 52174462Salfred 52274462Salfred CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 52374462Salfred __func__, dmat, flags, error, nsegs); 52474462Salfred 52574462Salfred if (error == EINPROGRESS) 52674462Salfred return (error); 52774462Salfred 528260250Sdelphij segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 52974462Salfred if (error) 53074462Salfred (*callback)(callback_arg, segs, 0, error); 53174462Salfred else 53274462Salfred (*callback)(callback_arg, segs, nsegs, 0); 53374462Salfred 53474462Salfred /* 53574462Salfred * Return ENOMEM to the caller so that it can pass it up the stack. 53674462Salfred * This error only happens when NOWAIT is set, so deferral is disabled. 53774462Salfred */ 53874462Salfred if (error == ENOMEM) 53974462Salfred return (error); 54074462Salfred 54174462Salfred return (0); 54274462Salfred} 54374462Salfred