1193260Sjhb/*- 2193260Sjhb * Copyright (c) 2008 Yahoo!, Inc. 3193260Sjhb * All rights reserved. 4193260Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 5193260Sjhb * 6193260Sjhb * Redistribution and use in source and binary forms, with or without 7193260Sjhb * modification, are permitted provided that the following conditions 8193260Sjhb * are met: 9193260Sjhb * 1. Redistributions of source code must retain the above copyright 10193260Sjhb * notice, this list of conditions and the following disclaimer. 11193260Sjhb * 2. Redistributions in binary form must reproduce the above copyright 12193260Sjhb * notice, this list of conditions and the following disclaimer in the 13193260Sjhb * documentation and/or other materials provided with the distribution. 14193260Sjhb * 3. Neither the name of the author nor the names of any co-contributors 15193260Sjhb * may be used to endorse or promote products derived from this software 16193260Sjhb * without specific prior written permission. 17193260Sjhb * 18193260Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19193260Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20193260Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21193260Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22193260Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23193260Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24193260Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25193260Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26193260Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27193260Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28193260Sjhb * SUCH DAMAGE. 29193260Sjhb */ 30193260Sjhb 31193260Sjhb#include <sys/cdefs.h> 32193260Sjhb__FBSDID("$FreeBSD: stable/11/sys/kern/subr_sglist.c 345039 2019-03-11 22:48:51Z jhb $"); 33193260Sjhb 34193260Sjhb#include <sys/param.h> 35193260Sjhb#include <sys/kernel.h> 36260581Sbryanv#include <sys/bio.h> 37193260Sjhb#include <sys/malloc.h> 38193260Sjhb#include <sys/mbuf.h> 39193260Sjhb#include <sys/proc.h> 40193260Sjhb#include <sys/sglist.h> 41193260Sjhb#include <sys/uio.h> 42193260Sjhb 43193260Sjhb#include <vm/vm.h> 44260581Sbryanv#include <vm/vm_page.h> 45193260Sjhb#include <vm/pmap.h> 46193260Sjhb#include <vm/vm_map.h> 47193260Sjhb 48193260Sjhb#include <sys/ktr.h> 49193260Sjhb 50193260Sjhbstatic MALLOC_DEFINE(M_SGLIST, "sglist", "scatter/gather lists"); 51193260Sjhb 52193260Sjhb/* 53196417Sjhb * Convenience macros to save the state of an sglist so it can be restored 54196417Sjhb * if an append attempt fails. Since sglist's only grow we only need to 55196417Sjhb * save the current count of segments and the length of the ending segment. 56196417Sjhb * Earlier segments will not be changed by an append, and the only change 57196417Sjhb * that can occur to the ending segment is that it can be extended. 58196417Sjhb */ 59196417Sjhbstruct sgsave { 60196417Sjhb u_short sg_nseg; 61196417Sjhb size_t ss_len; 62196417Sjhb}; 63196417Sjhb 64196417Sjhb#define SGLIST_SAVE(sg, sgsave) do { \ 65196417Sjhb (sgsave).sg_nseg = (sg)->sg_nseg; \ 66196417Sjhb if ((sgsave).sg_nseg > 0) \ 67196417Sjhb (sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \ 68196417Sjhb else \ 69196417Sjhb (sgsave).ss_len = 0; \ 70196417Sjhb} while (0) 71196417Sjhb 72196417Sjhb#define SGLIST_RESTORE(sg, sgsave) do { \ 73196417Sjhb (sg)->sg_nseg = (sgsave).sg_nseg; \ 74196417Sjhb if ((sgsave).sg_nseg > 0) \ 75196417Sjhb (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \ 76196417Sjhb} while (0) 77196417Sjhb 78196417Sjhb/* 79193260Sjhb * Append a single (paddr, len) to a sglist. sg is the list and ss is 80193260Sjhb * the current segment in the list. If we run out of segments then 81193260Sjhb * EFBIG will be returned. 82193260Sjhb */ 83193260Sjhbstatic __inline int 84193260Sjhb_sglist_append_range(struct sglist *sg, struct sglist_seg **ssp, 85193260Sjhb vm_paddr_t paddr, size_t len) 86193260Sjhb{ 87193260Sjhb struct sglist_seg *ss; 88193260Sjhb 89193260Sjhb ss = *ssp; 90193260Sjhb if (ss->ss_paddr + ss->ss_len == paddr) 91193260Sjhb ss->ss_len += len; 92193260Sjhb else { 93196417Sjhb if (sg->sg_nseg == sg->sg_maxseg) 94193260Sjhb return (EFBIG); 95193260Sjhb ss++; 96193260Sjhb ss->ss_paddr = paddr; 97193260Sjhb ss->ss_len = len; 98193260Sjhb sg->sg_nseg++; 99193260Sjhb *ssp = ss; 100193260Sjhb } 101193260Sjhb return (0); 102193260Sjhb} 103193260Sjhb 104193260Sjhb/* 105193260Sjhb * Worker routine to append a virtual address range (either kernel or 106193260Sjhb * user) to a scatter/gather list. 107193260Sjhb */ 108193260Sjhbstatic __inline int 109193260Sjhb_sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap, 110193260Sjhb size_t *donep) 111193260Sjhb{ 112193260Sjhb struct sglist_seg *ss; 113193260Sjhb vm_offset_t vaddr, offset; 114193260Sjhb vm_paddr_t paddr; 115193260Sjhb size_t seglen; 116193260Sjhb int error; 117193260Sjhb 118193260Sjhb if (donep) 119193260Sjhb *donep = 0; 120193260Sjhb if (len == 0) 121193260Sjhb return (0); 122193260Sjhb 123193260Sjhb /* Do the first page. It may have an offset. */ 124193260Sjhb vaddr = (vm_offset_t)buf; 125193260Sjhb offset = vaddr & PAGE_MASK; 126193260Sjhb if (pmap != NULL) 127193260Sjhb paddr = pmap_extract(pmap, vaddr); 128193260Sjhb else 129193260Sjhb paddr = pmap_kextract(vaddr); 130193260Sjhb seglen = MIN(len, PAGE_SIZE - offset); 131193260Sjhb if (sg->sg_nseg == 0) { 132193260Sjhb ss = sg->sg_segs; 133193260Sjhb ss->ss_paddr = paddr; 134193260Sjhb ss->ss_len = seglen; 135193260Sjhb sg->sg_nseg = 1; 136193260Sjhb } else { 137193260Sjhb ss = &sg->sg_segs[sg->sg_nseg - 1]; 138193260Sjhb error = _sglist_append_range(sg, &ss, paddr, seglen); 139196417Sjhb if (error) 140196417Sjhb return (error); 141193260Sjhb } 142196417Sjhb vaddr += seglen; 143196417Sjhb len -= seglen; 144196417Sjhb if (donep) 145196417Sjhb *donep += seglen; 146193260Sjhb 147196417Sjhb while (len > 0) { 148193260Sjhb seglen = MIN(len, PAGE_SIZE); 149193260Sjhb if (pmap != NULL) 150193260Sjhb paddr = pmap_extract(pmap, vaddr); 151193260Sjhb else 152193260Sjhb paddr = pmap_kextract(vaddr); 153193260Sjhb error = _sglist_append_range(sg, &ss, paddr, seglen); 154196417Sjhb if (error) 155196417Sjhb return (error); 156196417Sjhb vaddr += seglen; 157196417Sjhb len -= seglen; 158196417Sjhb if (donep) 159196417Sjhb *donep += seglen; 160193260Sjhb } 161193260Sjhb 162196417Sjhb return (0); 163193260Sjhb} 164193260Sjhb 165193260Sjhb/* 166193260Sjhb * Determine the number of scatter/gather list elements needed to 167193260Sjhb * describe a kernel virtual address range. 168193260Sjhb */ 169193260Sjhbint 170193260Sjhbsglist_count(void *buf, size_t len) 171193260Sjhb{ 172193260Sjhb vm_offset_t vaddr, vendaddr; 173193260Sjhb vm_paddr_t lastaddr, paddr; 174193260Sjhb int nsegs; 175193260Sjhb 176193260Sjhb if (len == 0) 177193260Sjhb return (0); 178193260Sjhb 179193260Sjhb vaddr = trunc_page((vm_offset_t)buf); 180193260Sjhb vendaddr = (vm_offset_t)buf + len; 181193260Sjhb nsegs = 1; 182193260Sjhb lastaddr = pmap_kextract(vaddr); 183193260Sjhb vaddr += PAGE_SIZE; 184193260Sjhb while (vaddr < vendaddr) { 185193260Sjhb paddr = pmap_kextract(vaddr); 186193260Sjhb if (lastaddr + PAGE_SIZE != paddr) 187193260Sjhb nsegs++; 188193260Sjhb lastaddr = paddr; 189193260Sjhb vaddr += PAGE_SIZE; 190193260Sjhb } 191193260Sjhb return (nsegs); 192193260Sjhb} 193193260Sjhb 194193260Sjhb/* 195300337Sjhb * Determine the number of scatter/gather list elements needed to 196300337Sjhb * describe a buffer backed by an array of VM pages. 197300337Sjhb */ 198300337Sjhbint 199300337Sjhbsglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len) 200300337Sjhb{ 201300337Sjhb vm_paddr_t lastaddr, paddr; 202300337Sjhb int i, nsegs; 203300337Sjhb 204300337Sjhb if (len == 0) 205300337Sjhb return (0); 206300337Sjhb 207300337Sjhb len += pgoff; 208300337Sjhb nsegs = 1; 209300337Sjhb lastaddr = VM_PAGE_TO_PHYS(m[0]); 210300337Sjhb for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) { 211300337Sjhb paddr = VM_PAGE_TO_PHYS(m[i]); 212300337Sjhb if (lastaddr + PAGE_SIZE != paddr) 213300337Sjhb nsegs++; 214300337Sjhb lastaddr = paddr; 215300337Sjhb } 216300337Sjhb return (nsegs); 217300337Sjhb} 218300337Sjhb 219300337Sjhb/* 220193260Sjhb * Allocate a scatter/gather list along with 'nsegs' segments. The 221193260Sjhb * 'mflags' parameters are the same as passed to malloc(9). The caller 222193260Sjhb * should use sglist_free() to free this list. 223193260Sjhb */ 224193260Sjhbstruct sglist * 225193260Sjhbsglist_alloc(int nsegs, int mflags) 226193260Sjhb{ 227193260Sjhb struct sglist *sg; 228193260Sjhb 229193260Sjhb sg = malloc(sizeof(struct sglist) + nsegs * sizeof(struct sglist_seg), 230193260Sjhb M_SGLIST, mflags); 231193260Sjhb if (sg == NULL) 232193260Sjhb return (NULL); 233193260Sjhb sglist_init(sg, nsegs, (struct sglist_seg *)(sg + 1)); 234193260Sjhb return (sg); 235193260Sjhb} 236193260Sjhb 237193260Sjhb/* 238193260Sjhb * Free a scatter/gather list allocated via sglist_allc(). 239193260Sjhb */ 240193260Sjhbvoid 241193260Sjhbsglist_free(struct sglist *sg) 242193260Sjhb{ 243193260Sjhb 244277759Sjhb if (sg == NULL) 245277759Sjhb return; 246277759Sjhb 247193260Sjhb if (refcount_release(&sg->sg_refs)) 248193260Sjhb free(sg, M_SGLIST); 249193260Sjhb} 250193260Sjhb 251193260Sjhb/* 252193260Sjhb * Append the segments to describe a single kernel virtual address 253193260Sjhb * range to a scatter/gather list. If there are insufficient 254193260Sjhb * segments, then this fails with EFBIG. 255193260Sjhb */ 256193260Sjhbint 257193260Sjhbsglist_append(struct sglist *sg, void *buf, size_t len) 258193260Sjhb{ 259196417Sjhb struct sgsave save; 260196417Sjhb int error; 261193260Sjhb 262193260Sjhb if (sg->sg_maxseg == 0) 263193260Sjhb return (EINVAL); 264196417Sjhb SGLIST_SAVE(sg, save); 265196417Sjhb error = _sglist_append_buf(sg, buf, len, NULL, NULL); 266196417Sjhb if (error) 267196417Sjhb SGLIST_RESTORE(sg, save); 268196417Sjhb return (error); 269193260Sjhb} 270193260Sjhb 271193260Sjhb/* 272260581Sbryanv * Append the segments to describe a bio's data to a scatter/gather list. 273260581Sbryanv * If there are insufficient segments, then this fails with EFBIG. 274260581Sbryanv * 275260581Sbryanv * NOTE: This function expects bio_bcount to be initialized. 276260581Sbryanv */ 277260581Sbryanvint 278260581Sbryanvsglist_append_bio(struct sglist *sg, struct bio *bp) 279260581Sbryanv{ 280300337Sjhb int error; 281260581Sbryanv 282300337Sjhb if ((bp->bio_flags & BIO_UNMAPPED) == 0) 283260581Sbryanv error = sglist_append(sg, bp->bio_data, bp->bio_bcount); 284300337Sjhb else 285300337Sjhb error = sglist_append_vmpages(sg, bp->bio_ma, 286300337Sjhb bp->bio_ma_offset, bp->bio_bcount); 287300337Sjhb return (error); 288260581Sbryanv} 289260581Sbryanv 290260581Sbryanv/* 291193260Sjhb * Append a single physical address range to a scatter/gather list. 292193260Sjhb * If there are insufficient segments, then this fails with EFBIG. 293193260Sjhb */ 294193260Sjhbint 295193260Sjhbsglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len) 296193260Sjhb{ 297193260Sjhb struct sglist_seg *ss; 298196417Sjhb struct sgsave save; 299196417Sjhb int error; 300193260Sjhb 301193260Sjhb if (sg->sg_maxseg == 0) 302193260Sjhb return (EINVAL); 303193260Sjhb if (len == 0) 304193260Sjhb return (0); 305193260Sjhb 306193260Sjhb if (sg->sg_nseg == 0) { 307193260Sjhb sg->sg_segs[0].ss_paddr = paddr; 308193260Sjhb sg->sg_segs[0].ss_len = len; 309193260Sjhb sg->sg_nseg = 1; 310193260Sjhb return (0); 311193260Sjhb } 312193260Sjhb ss = &sg->sg_segs[sg->sg_nseg - 1]; 313196417Sjhb SGLIST_SAVE(sg, save); 314196417Sjhb error = _sglist_append_range(sg, &ss, paddr, len); 315196417Sjhb if (error) 316196417Sjhb SGLIST_RESTORE(sg, save); 317196417Sjhb return (error); 318193260Sjhb} 319193260Sjhb 320193260Sjhb/* 321193260Sjhb * Append the segments that describe a single mbuf chain to a 322193260Sjhb * scatter/gather list. If there are insufficient segments, then this 323193260Sjhb * fails with EFBIG. 324193260Sjhb */ 325193260Sjhbint 326193260Sjhbsglist_append_mbuf(struct sglist *sg, struct mbuf *m0) 327193260Sjhb{ 328196417Sjhb struct sgsave save; 329193260Sjhb struct mbuf *m; 330193260Sjhb int error; 331193260Sjhb 332193260Sjhb if (sg->sg_maxseg == 0) 333193260Sjhb return (EINVAL); 334193260Sjhb 335193260Sjhb error = 0; 336196417Sjhb SGLIST_SAVE(sg, save); 337193260Sjhb for (m = m0; m != NULL; m = m->m_next) { 338193260Sjhb if (m->m_len > 0) { 339193260Sjhb error = sglist_append(sg, m->m_data, m->m_len); 340196417Sjhb if (error) { 341196417Sjhb SGLIST_RESTORE(sg, save); 342193260Sjhb return (error); 343196417Sjhb } 344193260Sjhb } 345193260Sjhb } 346193260Sjhb return (0); 347193260Sjhb} 348193260Sjhb 349193260Sjhb/* 350300337Sjhb * Append the segments that describe a buffer spanning an array of VM 351300337Sjhb * pages. The buffer begins at an offset of 'pgoff' in the first 352300337Sjhb * page. 353300337Sjhb */ 354300337Sjhbint 355300337Sjhbsglist_append_vmpages(struct sglist *sg, vm_page_t *m, size_t pgoff, 356300337Sjhb size_t len) 357300337Sjhb{ 358300337Sjhb struct sgsave save; 359300337Sjhb struct sglist_seg *ss; 360300337Sjhb vm_paddr_t paddr; 361300337Sjhb size_t seglen; 362300337Sjhb int error, i; 363300337Sjhb 364300337Sjhb if (sg->sg_maxseg == 0) 365300337Sjhb return (EINVAL); 366300337Sjhb if (len == 0) 367300337Sjhb return (0); 368300337Sjhb 369300337Sjhb SGLIST_SAVE(sg, save); 370300337Sjhb i = 0; 371300337Sjhb if (sg->sg_nseg == 0) { 372300337Sjhb seglen = min(PAGE_SIZE - pgoff, len); 373300337Sjhb sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff; 374300337Sjhb sg->sg_segs[0].ss_len = seglen; 375300337Sjhb sg->sg_nseg = 1; 376300337Sjhb pgoff = 0; 377300337Sjhb len -= seglen; 378300337Sjhb i++; 379300337Sjhb } 380300337Sjhb ss = &sg->sg_segs[sg->sg_nseg - 1]; 381300337Sjhb for (; len > 0; i++, len -= seglen) { 382300337Sjhb seglen = min(PAGE_SIZE - pgoff, len); 383300337Sjhb paddr = VM_PAGE_TO_PHYS(m[i]) + pgoff; 384300337Sjhb error = _sglist_append_range(sg, &ss, paddr, seglen); 385300337Sjhb if (error) { 386300337Sjhb SGLIST_RESTORE(sg, save); 387300337Sjhb return (error); 388300337Sjhb } 389300337Sjhb pgoff = 0; 390300337Sjhb } 391300337Sjhb return (0); 392300337Sjhb} 393300337Sjhb 394300337Sjhb/* 395193260Sjhb * Append the segments that describe a single user address range to a 396193260Sjhb * scatter/gather list. If there are insufficient segments, then this 397193260Sjhb * fails with EFBIG. 398193260Sjhb */ 399193260Sjhbint 400193260Sjhbsglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td) 401193260Sjhb{ 402196417Sjhb struct sgsave save; 403196417Sjhb int error; 404193260Sjhb 405193260Sjhb if (sg->sg_maxseg == 0) 406193260Sjhb return (EINVAL); 407196417Sjhb SGLIST_SAVE(sg, save); 408196417Sjhb error = _sglist_append_buf(sg, buf, len, 409196417Sjhb vmspace_pmap(td->td_proc->p_vmspace), NULL); 410196417Sjhb if (error) 411196417Sjhb SGLIST_RESTORE(sg, save); 412196417Sjhb return (error); 413193260Sjhb} 414193260Sjhb 415193260Sjhb/* 416345039Sjhb * Append a subset of an existing scatter/gather list 'source' to a 417345039Sjhb * the scatter/gather list 'sg'. If there are insufficient segments, 418345039Sjhb * then this fails with EFBIG. 419345039Sjhb */ 420345039Sjhbint 421345039Sjhbsglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset, 422345039Sjhb size_t length) 423345039Sjhb{ 424345039Sjhb struct sgsave save; 425345039Sjhb struct sglist_seg *ss; 426345039Sjhb size_t seglen; 427345039Sjhb int error, i; 428345039Sjhb 429345039Sjhb if (sg->sg_maxseg == 0 || length == 0) 430345039Sjhb return (EINVAL); 431345039Sjhb SGLIST_SAVE(sg, save); 432345039Sjhb error = EINVAL; 433345039Sjhb ss = &sg->sg_segs[sg->sg_nseg - 1]; 434345039Sjhb for (i = 0; i < source->sg_nseg; i++) { 435345039Sjhb if (offset >= source->sg_segs[i].ss_len) { 436345039Sjhb offset -= source->sg_segs[i].ss_len; 437345039Sjhb continue; 438345039Sjhb } 439345039Sjhb seglen = source->sg_segs[i].ss_len - offset; 440345039Sjhb if (seglen > length) 441345039Sjhb seglen = length; 442345039Sjhb error = _sglist_append_range(sg, &ss, 443345039Sjhb source->sg_segs[i].ss_paddr + offset, seglen); 444345039Sjhb if (error) 445345039Sjhb break; 446345039Sjhb offset = 0; 447345039Sjhb length -= seglen; 448345039Sjhb if (length == 0) 449345039Sjhb break; 450345039Sjhb } 451345039Sjhb if (length != 0) 452345039Sjhb error = EINVAL; 453345039Sjhb if (error) 454345039Sjhb SGLIST_RESTORE(sg, save); 455345039Sjhb return (error); 456345039Sjhb} 457345039Sjhb 458345039Sjhb/* 459193260Sjhb * Append the segments that describe a single uio to a scatter/gather 460193260Sjhb * list. If there are insufficient segments, then this fails with 461193260Sjhb * EFBIG. 462193260Sjhb */ 463193260Sjhbint 464193260Sjhbsglist_append_uio(struct sglist *sg, struct uio *uio) 465193260Sjhb{ 466193260Sjhb struct iovec *iov; 467196417Sjhb struct sgsave save; 468193260Sjhb size_t resid, minlen; 469193260Sjhb pmap_t pmap; 470193260Sjhb int error, i; 471193260Sjhb 472193260Sjhb if (sg->sg_maxseg == 0) 473193260Sjhb return (EINVAL); 474193260Sjhb 475193260Sjhb resid = uio->uio_resid; 476193260Sjhb iov = uio->uio_iov; 477193260Sjhb 478193260Sjhb if (uio->uio_segflg == UIO_USERSPACE) { 479193260Sjhb KASSERT(uio->uio_td != NULL, 480193260Sjhb ("sglist_append_uio: USERSPACE but no thread")); 481193260Sjhb pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 482193260Sjhb } else 483193260Sjhb pmap = NULL; 484193260Sjhb 485193260Sjhb error = 0; 486196417Sjhb SGLIST_SAVE(sg, save); 487193260Sjhb for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 488193260Sjhb /* 489193260Sjhb * Now at the first iovec to load. Load each iovec 490193260Sjhb * until we have exhausted the residual count. 491193260Sjhb */ 492193260Sjhb minlen = MIN(resid, iov[i].iov_len); 493193260Sjhb if (minlen > 0) { 494193260Sjhb error = _sglist_append_buf(sg, iov[i].iov_base, minlen, 495193260Sjhb pmap, NULL); 496196417Sjhb if (error) { 497196417Sjhb SGLIST_RESTORE(sg, save); 498193260Sjhb return (error); 499196417Sjhb } 500193260Sjhb resid -= minlen; 501193260Sjhb } 502193260Sjhb } 503193260Sjhb return (0); 504193260Sjhb} 505193260Sjhb 506193260Sjhb/* 507193260Sjhb * Append the segments that describe at most 'resid' bytes from a 508193260Sjhb * single uio to a scatter/gather list. If there are insufficient 509193260Sjhb * segments, then only the amount that fits is appended. 510193260Sjhb */ 511193260Sjhbint 512196404Sjhbsglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid) 513193260Sjhb{ 514193260Sjhb struct iovec *iov; 515193260Sjhb size_t done; 516193260Sjhb pmap_t pmap; 517193260Sjhb int error, len; 518193260Sjhb 519193260Sjhb if (sg->sg_maxseg == 0) 520193260Sjhb return (EINVAL); 521193260Sjhb 522193260Sjhb if (uio->uio_segflg == UIO_USERSPACE) { 523193260Sjhb KASSERT(uio->uio_td != NULL, 524193260Sjhb ("sglist_consume_uio: USERSPACE but no thread")); 525193260Sjhb pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 526193260Sjhb } else 527193260Sjhb pmap = NULL; 528193260Sjhb 529193260Sjhb error = 0; 530193260Sjhb while (resid > 0 && uio->uio_resid) { 531193260Sjhb iov = uio->uio_iov; 532193260Sjhb len = iov->iov_len; 533193260Sjhb if (len == 0) { 534193260Sjhb uio->uio_iov++; 535193260Sjhb uio->uio_iovcnt--; 536193260Sjhb continue; 537193260Sjhb } 538193260Sjhb if (len > resid) 539193260Sjhb len = resid; 540193260Sjhb 541193260Sjhb /* 542193260Sjhb * Try to append this iovec. If we run out of room, 543193260Sjhb * then break out of the loop. 544193260Sjhb */ 545193260Sjhb error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done); 546193260Sjhb iov->iov_base = (char *)iov->iov_base + done; 547193260Sjhb iov->iov_len -= done; 548193260Sjhb uio->uio_resid -= done; 549193260Sjhb uio->uio_offset += done; 550193260Sjhb resid -= done; 551193260Sjhb if (error) 552193260Sjhb break; 553193260Sjhb } 554193260Sjhb return (0); 555193260Sjhb} 556193260Sjhb 557193260Sjhb/* 558193260Sjhb * Allocate and populate a scatter/gather list to describe a single 559193260Sjhb * kernel virtual address range. 560193260Sjhb */ 561193260Sjhbstruct sglist * 562193260Sjhbsglist_build(void *buf, size_t len, int mflags) 563193260Sjhb{ 564193260Sjhb struct sglist *sg; 565193260Sjhb int nsegs; 566193260Sjhb 567193260Sjhb if (len == 0) 568193260Sjhb return (NULL); 569193260Sjhb 570193260Sjhb nsegs = sglist_count(buf, len); 571193260Sjhb sg = sglist_alloc(nsegs, mflags); 572193260Sjhb if (sg == NULL) 573193260Sjhb return (NULL); 574193260Sjhb if (sglist_append(sg, buf, len) != 0) { 575193260Sjhb sglist_free(sg); 576193260Sjhb return (NULL); 577193260Sjhb } 578193260Sjhb return (sg); 579193260Sjhb} 580193260Sjhb 581193260Sjhb/* 582193260Sjhb * Clone a new copy of a scatter/gather list. 583193260Sjhb */ 584193260Sjhbstruct sglist * 585193260Sjhbsglist_clone(struct sglist *sg, int mflags) 586193260Sjhb{ 587193260Sjhb struct sglist *new; 588193260Sjhb 589193260Sjhb if (sg == NULL) 590193260Sjhb return (NULL); 591193260Sjhb new = sglist_alloc(sg->sg_maxseg, mflags); 592193260Sjhb if (new == NULL) 593193260Sjhb return (NULL); 594196417Sjhb new->sg_nseg = sg->sg_nseg; 595193260Sjhb bcopy(sg->sg_segs, new->sg_segs, sizeof(struct sglist_seg) * 596193260Sjhb sg->sg_nseg); 597193260Sjhb return (new); 598193260Sjhb} 599193260Sjhb 600193260Sjhb/* 601193260Sjhb * Calculate the total length of the segments described in a 602193260Sjhb * scatter/gather list. 603193260Sjhb */ 604193260Sjhbsize_t 605193260Sjhbsglist_length(struct sglist *sg) 606193260Sjhb{ 607193260Sjhb size_t space; 608193260Sjhb int i; 609193260Sjhb 610193260Sjhb space = 0; 611193260Sjhb for (i = 0; i < sg->sg_nseg; i++) 612193260Sjhb space += sg->sg_segs[i].ss_len; 613193260Sjhb return (space); 614193260Sjhb} 615193260Sjhb 616193260Sjhb/* 617193260Sjhb * Split a scatter/gather list into two lists. The scatter/gather 618193260Sjhb * entries for the first 'length' bytes of the 'original' list are 619193260Sjhb * stored in the '*head' list and are removed from 'original'. 620193260Sjhb * 621193260Sjhb * If '*head' is NULL, then a new list will be allocated using 622193260Sjhb * 'mflags'. If M_NOWAIT is specified and the allocation fails, 623193260Sjhb * ENOMEM will be returned. 624193260Sjhb * 625193260Sjhb * If '*head' is not NULL, it should point to an empty sglist. If it 626193260Sjhb * does not have enough room for the remaining space, then EFBIG will 627193260Sjhb * be returned. If '*head' is not empty, then EINVAL will be 628193260Sjhb * returned. 629193260Sjhb * 630193260Sjhb * If 'original' is shared (refcount > 1), then EDOOFUS will be 631193260Sjhb * returned. 632193260Sjhb */ 633193260Sjhbint 634193260Sjhbsglist_split(struct sglist *original, struct sglist **head, size_t length, 635193260Sjhb int mflags) 636193260Sjhb{ 637193260Sjhb struct sglist *sg; 638193260Sjhb size_t space, split; 639193260Sjhb int count, i; 640193260Sjhb 641193260Sjhb if (original->sg_refs > 1) 642193260Sjhb return (EDOOFUS); 643193260Sjhb 644193260Sjhb /* Figure out how big of a sglist '*head' has to hold. */ 645193260Sjhb count = 0; 646193260Sjhb space = 0; 647193260Sjhb split = 0; 648193260Sjhb for (i = 0; i < original->sg_nseg; i++) { 649193260Sjhb space += original->sg_segs[i].ss_len; 650193260Sjhb count++; 651193260Sjhb if (space >= length) { 652193260Sjhb /* 653193260Sjhb * If 'length' falls in the middle of a 654193260Sjhb * scatter/gather list entry, then 'split' 655193260Sjhb * holds how much of that entry will remain in 656193260Sjhb * 'original'. 657193260Sjhb */ 658193260Sjhb split = space - length; 659193260Sjhb break; 660193260Sjhb } 661193260Sjhb } 662193260Sjhb 663193260Sjhb /* Nothing to do, so leave head empty. */ 664193260Sjhb if (count == 0) 665193260Sjhb return (0); 666193260Sjhb 667193260Sjhb if (*head == NULL) { 668193260Sjhb sg = sglist_alloc(count, mflags); 669193260Sjhb if (sg == NULL) 670193260Sjhb return (ENOMEM); 671193260Sjhb *head = sg; 672193260Sjhb } else { 673193260Sjhb sg = *head; 674193260Sjhb if (sg->sg_maxseg < count) 675193260Sjhb return (EFBIG); 676193260Sjhb if (sg->sg_nseg != 0) 677193260Sjhb return (EINVAL); 678193260Sjhb } 679193260Sjhb 680193260Sjhb /* Copy 'count' entries to 'sg' from 'original'. */ 681193260Sjhb bcopy(original->sg_segs, sg->sg_segs, count * 682193260Sjhb sizeof(struct sglist_seg)); 683193260Sjhb sg->sg_nseg = count; 684193260Sjhb 685193260Sjhb /* 686193260Sjhb * If we had to split a list entry, fixup the last entry in 687193260Sjhb * 'sg' and the new first entry in 'original'. We also 688193260Sjhb * decrement 'count' by 1 since we will only be removing 689193260Sjhb * 'count - 1' segments from 'original' now. 690193260Sjhb */ 691193260Sjhb if (split != 0) { 692193260Sjhb count--; 693193260Sjhb sg->sg_segs[count].ss_len -= split; 694193260Sjhb original->sg_segs[count].ss_paddr = 695193260Sjhb sg->sg_segs[count].ss_paddr + split; 696193260Sjhb original->sg_segs[count].ss_len = split; 697193260Sjhb } 698193260Sjhb 699193260Sjhb /* Trim 'count' entries from the front of 'original'. */ 700193260Sjhb original->sg_nseg -= count; 701193260Sjhb bcopy(original->sg_segs + count, original->sg_segs, count * 702193260Sjhb sizeof(struct sglist_seg)); 703193260Sjhb return (0); 704193260Sjhb} 705193260Sjhb 706193260Sjhb/* 707193260Sjhb * Append the scatter/gather list elements in 'second' to the 708193260Sjhb * scatter/gather list 'first'. If there is not enough space in 709193260Sjhb * 'first', EFBIG is returned. 710193260Sjhb */ 711193260Sjhbint 712193260Sjhbsglist_join(struct sglist *first, struct sglist *second) 713193260Sjhb{ 714193260Sjhb struct sglist_seg *flast, *sfirst; 715193260Sjhb int append; 716193260Sjhb 717193260Sjhb /* If 'second' is empty, there is nothing to do. */ 718193260Sjhb if (second->sg_nseg == 0) 719193260Sjhb return (0); 720193260Sjhb 721193260Sjhb /* 722193260Sjhb * If the first entry in 'second' can be appended to the last entry 723193260Sjhb * in 'first' then set append to '1'. 724193260Sjhb */ 725193260Sjhb append = 0; 726193260Sjhb flast = &first->sg_segs[first->sg_nseg - 1]; 727193260Sjhb sfirst = &second->sg_segs[0]; 728193260Sjhb if (first->sg_nseg != 0 && 729193260Sjhb flast->ss_paddr + flast->ss_len == sfirst->ss_paddr) 730193260Sjhb append = 1; 731193260Sjhb 732193260Sjhb /* Make sure 'first' has enough room. */ 733193260Sjhb if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg) 734193260Sjhb return (EFBIG); 735193260Sjhb 736193260Sjhb /* Merge last in 'first' and first in 'second' if needed. */ 737193260Sjhb if (append) 738193260Sjhb flast->ss_len += sfirst->ss_len; 739193260Sjhb 740193260Sjhb /* Append new segments from 'second' to 'first'. */ 741193260Sjhb bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append, 742193260Sjhb (second->sg_nseg - append) * sizeof(struct sglist_seg)); 743193260Sjhb first->sg_nseg += second->sg_nseg - append; 744193260Sjhb sglist_reset(second); 745193260Sjhb return (0); 746193260Sjhb} 747193260Sjhb 748193260Sjhb/* 749193260Sjhb * Generate a new scatter/gather list from a range of an existing 750193260Sjhb * scatter/gather list. The 'offset' and 'length' parameters specify 751193260Sjhb * the logical range of the 'original' list to extract. If that range 752193260Sjhb * is not a subset of the length of 'original', then EINVAL is 753193260Sjhb * returned. The new scatter/gather list is stored in '*slice'. 754193260Sjhb * 755193260Sjhb * If '*slice' is NULL, then a new list will be allocated using 756193260Sjhb * 'mflags'. If M_NOWAIT is specified and the allocation fails, 757193260Sjhb * ENOMEM will be returned. 758193260Sjhb * 759193260Sjhb * If '*slice' is not NULL, it should point to an empty sglist. If it 760193260Sjhb * does not have enough room for the remaining space, then EFBIG will 761193260Sjhb * be returned. If '*slice' is not empty, then EINVAL will be 762193260Sjhb * returned. 763193260Sjhb */ 764193260Sjhbint 765193260Sjhbsglist_slice(struct sglist *original, struct sglist **slice, size_t offset, 766193260Sjhb size_t length, int mflags) 767193260Sjhb{ 768193260Sjhb struct sglist *sg; 769193260Sjhb size_t space, end, foffs, loffs; 770193260Sjhb int count, i, fseg; 771193260Sjhb 772193260Sjhb /* Nothing to do. */ 773193260Sjhb if (length == 0) 774193260Sjhb return (0); 775193260Sjhb 776193260Sjhb /* Figure out how many segments '*slice' needs to have. */ 777193260Sjhb end = offset + length; 778193260Sjhb space = 0; 779193260Sjhb count = 0; 780193260Sjhb fseg = 0; 781193260Sjhb foffs = loffs = 0; 782193260Sjhb for (i = 0; i < original->sg_nseg; i++) { 783193260Sjhb space += original->sg_segs[i].ss_len; 784193260Sjhb if (space > offset) { 785193260Sjhb /* 786193260Sjhb * When we hit the first segment, store its index 787193260Sjhb * in 'fseg' and the offset into the first segment 788193260Sjhb * of 'offset' in 'foffs'. 789193260Sjhb */ 790193260Sjhb if (count == 0) { 791193260Sjhb fseg = i; 792193260Sjhb foffs = offset - (space - 793193260Sjhb original->sg_segs[i].ss_len); 794193260Sjhb CTR1(KTR_DEV, "sglist_slice: foffs = %08lx", 795193260Sjhb foffs); 796193260Sjhb } 797193260Sjhb count++; 798193260Sjhb 799193260Sjhb /* 800193260Sjhb * When we hit the last segment, break out of 801193260Sjhb * the loop. Store the amount of extra space 802193260Sjhb * at the end of this segment in 'loffs'. 803193260Sjhb */ 804193260Sjhb if (space >= end) { 805193260Sjhb loffs = space - end; 806193260Sjhb CTR1(KTR_DEV, "sglist_slice: loffs = %08lx", 807193260Sjhb loffs); 808193260Sjhb break; 809193260Sjhb } 810193260Sjhb } 811193260Sjhb } 812193260Sjhb 813193260Sjhb /* If we never hit 'end', then 'length' ran off the end, so fail. */ 814193260Sjhb if (space < end) 815193260Sjhb return (EINVAL); 816193260Sjhb 817193260Sjhb if (*slice == NULL) { 818193260Sjhb sg = sglist_alloc(count, mflags); 819193260Sjhb if (sg == NULL) 820193260Sjhb return (ENOMEM); 821193260Sjhb *slice = sg; 822193260Sjhb } else { 823193260Sjhb sg = *slice; 824193260Sjhb if (sg->sg_maxseg < count) 825193260Sjhb return (EFBIG); 826193260Sjhb if (sg->sg_nseg != 0) 827193260Sjhb return (EINVAL); 828193260Sjhb } 829193260Sjhb 830193260Sjhb /* 831193260Sjhb * Copy over 'count' segments from 'original' starting at 832193260Sjhb * 'fseg' to 'sg'. 833193260Sjhb */ 834193260Sjhb bcopy(original->sg_segs + fseg, sg->sg_segs, 835193260Sjhb count * sizeof(struct sglist_seg)); 836193260Sjhb sg->sg_nseg = count; 837193260Sjhb 838193260Sjhb /* Fixup first and last segments if needed. */ 839193260Sjhb if (foffs != 0) { 840193260Sjhb sg->sg_segs[0].ss_paddr += foffs; 841193260Sjhb sg->sg_segs[0].ss_len -= foffs; 842193260Sjhb CTR2(KTR_DEV, "sglist_slice seg[0]: %08lx:%08lx", 843193260Sjhb (long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len); 844193260Sjhb } 845193260Sjhb if (loffs != 0) { 846193260Sjhb sg->sg_segs[count - 1].ss_len -= loffs; 847193260Sjhb CTR2(KTR_DEV, "sglist_slice seg[%d]: len %08x", count - 1, 848193260Sjhb sg->sg_segs[count - 1].ss_len); 849193260Sjhb } 850193260Sjhb return (0); 851193260Sjhb} 852