subr_uio.c revision 83420
119370Spst/* 219370Spst * Copyright (c) 1982, 1986, 1991, 1993 3130803Smarcel * The Regents of the University of California. All rights reserved. 4130803Smarcel * (c) UNIX System Laboratories, Inc. 5130803Smarcel * All or some portions of this file are derived from material licensed 6130803Smarcel * to the University of California by American Telephone and Telegraph 798944Sobrien * Co. or Unix System Laboratories, Inc. and are reproduced herein with 819370Spst * the permission of UNIX System Laboratories, Inc. 998944Sobrien * 1098944Sobrien * Redistribution and use in source and binary forms, with or without 1198944Sobrien * modification, are permitted provided that the following conditions 1298944Sobrien * are met: 1319370Spst * 1. Redistributions of source code must retain the above copyright 1498944Sobrien * notice, this list of conditions and the following disclaimer. 1598944Sobrien * 2. Redistributions in binary form must reproduce the above copyright 1698944Sobrien * notice, this list of conditions and the following disclaimer in the 1798944Sobrien * documentation and/or other materials provided with the distribution. 1819370Spst * 3. All advertising materials mentioning features or use of this software 1998944Sobrien * must display the following acknowledgement: 2098944Sobrien * This product includes software developed by the University of 2198944Sobrien * California, Berkeley and its contributors. 2298944Sobrien * 4. Neither the name of the University nor the names of its contributors 2319370Spst * may be used to endorse or promote products derived from this software 2419370Spst * without specific prior written permission. 2519370Spst * 2619370Spst * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2719370Spst * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2819370Spst * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2919370Spst * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 3019370Spst * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 3119370Spst * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3219370Spst * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3398944Sobrien * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3446283Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3598944Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3698944Sobrien * SUCH DAMAGE. 37130803Smarcel * 38130803Smarcel * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94 3919370Spst * $FreeBSD: head/sys/kern/kern_subr.c 83420 2001-09-13 22:33:37Z jhb $ 4019370Spst */ 4119370Spst 4219370Spst#include <sys/param.h> 4319370Spst#include <sys/systm.h> 4419370Spst#include <sys/kernel.h> 4519370Spst#include <sys/ktr.h> 4619370Spst#include <sys/lock.h> 4719370Spst#include <sys/mutex.h> 4898944Sobrien#include <sys/proc.h> 4919370Spst#include <sys/malloc.h> 5019370Spst#include <sys/resourcevar.h> 5119370Spst#include <sys/sysctl.h> 52130803Smarcel#include <sys/vnode.h> 5319370Spst 5419370Spst#include <vm/vm.h> 55130803Smarcel#include <vm/vm_page.h> 56130803Smarcel#include <vm/vm_map.h> 57130803Smarcel 5819370Spststatic void uio_yield __P((void)); 5919370Spst 6019370SpstSYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV, 6119370Spst "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)"); 62130803Smarcel 6319370Spstint 6419370Spstuiomove(cp, n, uio) 6519370Spst register caddr_t cp; 6698944Sobrien register int n; 6719370Spst register struct uio *uio; 6819370Spst{ 6919370Spst struct thread *td = curthread; 7098944Sobrien register struct iovec *iov; 7119370Spst u_int cnt; 7219370Spst int error = 0; 7319370Spst int save = 0; 7419370Spst 7519370Spst KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, 7619370Spst ("uiomove: mode")); 7719370Spst KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, 7898944Sobrien ("uiomove proc")); 7919370Spst 8019370Spst if (td) { 8119370Spst mtx_lock_spin(&sched_lock); 8219370Spst save = td->td_flags & TDF_DEADLKTREAT; 8319370Spst td->td_flags |= TDF_DEADLKTREAT; 8419370Spst mtx_unlock_spin(&sched_lock); 8546283Sdfr } 86130803Smarcel 8719370Spst while (n > 0 && uio->uio_resid) { 8846283Sdfr iov = uio->uio_iov; 89130803Smarcel cnt = iov->iov_len; 90130803Smarcel if (cnt == 0) { 91130803Smarcel uio->uio_iov++; 9219370Spst uio->uio_iovcnt--; 9346283Sdfr continue; 9419370Spst } 9519370Spst if (cnt > n) 96130803Smarcel cnt = n; 9719370Spst 9819370Spst switch (uio->uio_segflg) { 9919370Spst 10019370Spst case UIO_USERSPACE: 10198944Sobrien case UIO_USERISPACE: 10219370Spst if (ticks - PCPU_GET(switchticks) >= hogticks) 10319370Spst uio_yield(); 10419370Spst if (uio->uio_rw == UIO_READ) 10519370Spst error = copyout(cp, iov->iov_base, cnt); 10619370Spst else 10719370Spst error = copyin(iov->iov_base, cp, cnt); 10819370Spst if (error) 10919370Spst break; 11019370Spst break; 11119370Spst 11219370Spst case UIO_SYSSPACE: 11319370Spst if (uio->uio_rw == UIO_READ) 11419370Spst bcopy((caddr_t)cp, iov->iov_base, cnt); 11519370Spst else 11619370Spst bcopy(iov->iov_base, (caddr_t)cp, cnt); 11719370Spst break; 11819370Spst case UIO_NOCOPY: 11919370Spst break; 120130803Smarcel } 12119370Spst iov->iov_base += cnt; 12219370Spst iov->iov_len -= cnt; 12319370Spst uio->uio_resid -= cnt; 12419370Spst uio->uio_offset += cnt; 12519370Spst cp += cnt; 12698944Sobrien n -= cnt; 12719370Spst } 12819370Spst if (td != curthread) printf("uiomove: IT CHANGED!"); 12919370Spst td = curthread; /* Might things have changed in copyin/copyout? */ 13019370Spst if (td) { 13119370Spst mtx_lock_spin(&sched_lock); 13219370Spst td->td_flags = (td->td_flags & ~TDF_DEADLKTREAT) | save; 13319370Spst mtx_unlock_spin(&sched_lock); 13419370Spst } 13519370Spst return (error); 13619370Spst} 13719370Spst 13819370Spstint 13919370Spstuiomoveco(cp, n, uio, obj) 14019370Spst caddr_t cp; 14119370Spst int n; 14219370Spst struct uio *uio; 14319370Spst struct vm_object *obj; 14419370Spst{ 14519370Spst struct iovec *iov; 14619370Spst u_int cnt; 14719370Spst int error; 14819370Spst 14919370Spst KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, 15019370Spst ("uiomoveco: mode")); 15119370Spst KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, 15219370Spst ("uiomoveco proc")); 15319370Spst 15419370Spst while (n > 0 && uio->uio_resid) { 15519370Spst iov = uio->uio_iov; 15619370Spst cnt = iov->iov_len; 15719370Spst if (cnt == 0) { 15819370Spst uio->uio_iov++; 15919370Spst uio->uio_iovcnt--; 16019370Spst continue; 16119370Spst } 16219370Spst if (cnt > n) 16398944Sobrien cnt = n; 16498944Sobrien 16598944Sobrien switch (uio->uio_segflg) { 16698944Sobrien 167130803Smarcel case UIO_USERSPACE: 16898944Sobrien case UIO_USERISPACE: 16998944Sobrien if (ticks - PCPU_GET(switchticks) >= hogticks) 17098944Sobrien uio_yield(); 17198944Sobrien if (uio->uio_rw == UIO_READ) { 17298944Sobrien#ifdef ENABLE_VFS_IOOPT 17398944Sobrien if (vfs_ioopt && ((cnt & PAGE_MASK) == 0) && 17498944Sobrien ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) && 17598944Sobrien ((uio->uio_offset & PAGE_MASK) == 0) && 17698944Sobrien ((((intptr_t) cp) & PAGE_MASK) == 0)) { 17798944Sobrien error = vm_uiomove(&curproc->p_vmspace->vm_map, obj, 17898944Sobrien uio->uio_offset, cnt, 17919370Spst (vm_offset_t) iov->iov_base, NULL); 18098944Sobrien } else 18119370Spst#endif 18219370Spst { 18398944Sobrien error = copyout(cp, iov->iov_base, cnt); 18419370Spst } 18519370Spst } else { 18619370Spst error = copyin(iov->iov_base, cp, cnt); 18719370Spst } 18898944Sobrien if (error) 18919370Spst return (error); 19019370Spst break; 19119370Spst 19219370Spst case UIO_SYSSPACE: 19319370Spst if (uio->uio_rw == UIO_READ) 19419370Spst bcopy((caddr_t)cp, iov->iov_base, cnt); 19519370Spst else 19619370Spst bcopy(iov->iov_base, (caddr_t)cp, cnt); 19719370Spst break; 19819370Spst case UIO_NOCOPY: 19919370Spst break; 20019370Spst } 20119370Spst iov->iov_base += cnt; 20219370Spst iov->iov_len -= cnt; 20319370Spst uio->uio_resid -= cnt; 20419370Spst uio->uio_offset += cnt; 20519370Spst cp += cnt; 20619370Spst n -= cnt; 20798944Sobrien } 20819370Spst return (0); 20919370Spst} 21098944Sobrien 21119370Spst#ifdef ENABLE_VFS_IOOPT 21219370Spst 21319370Spstint 21419370Spstuioread(n, uio, obj, nread) 21598944Sobrien int n; 21619370Spst struct uio *uio; 21719370Spst struct vm_object *obj; 21819370Spst int *nread; 21919370Spst{ 22019370Spst int npagesmoved; 22119370Spst struct iovec *iov; 22219370Spst u_int cnt, tcnt; 22319370Spst int error; 22419370Spst 22519370Spst *nread = 0; 22619370Spst if (vfs_ioopt < 2) 22719370Spst return 0; 22819370Spst 22919370Spst error = 0; 23019370Spst 23119370Spst while (n > 0 && uio->uio_resid) { 23219370Spst iov = uio->uio_iov; 23398944Sobrien cnt = iov->iov_len; 23498944Sobrien if (cnt == 0) { 23519370Spst uio->uio_iov++; 23698944Sobrien uio->uio_iovcnt--; 23719370Spst continue; 23898944Sobrien } 23998944Sobrien if (cnt > n) 24098944Sobrien cnt = n; 24198944Sobrien 24298944Sobrien if ((uio->uio_segflg == UIO_USERSPACE) && 24319370Spst ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) && 24498944Sobrien ((uio->uio_offset & PAGE_MASK) == 0) ) { 24519370Spst 24619370Spst if (cnt < PAGE_SIZE) 24719370Spst break; 24819370Spst 249130803Smarcel cnt &= ~PAGE_MASK; 250130803Smarcel 251130803Smarcel if (ticks - PCPU_GET(switchticks) >= hogticks) 25219370Spst uio_yield(); 253130803Smarcel error = vm_uiomove(&curproc->p_vmspace->vm_map, obj, 254130803Smarcel uio->uio_offset, cnt, 25546283Sdfr (vm_offset_t) iov->iov_base, &npagesmoved); 25698944Sobrien 257130803Smarcel if (npagesmoved == 0) 25819370Spst break; 25919370Spst 26019370Spst tcnt = npagesmoved * PAGE_SIZE; 26198944Sobrien cnt = tcnt; 262130803Smarcel 263130803Smarcel if (error) 26419370Spst break; 26519370Spst 266130803Smarcel iov->iov_base += cnt; 267130803Smarcel iov->iov_len -= cnt; 268130803Smarcel uio->uio_resid -= cnt; 269130803Smarcel uio->uio_offset += cnt; 27019370Spst *nread += cnt; 271130803Smarcel n -= cnt; 272130803Smarcel } else { 273130803Smarcel break; 274130803Smarcel } 275130803Smarcel } 276130803Smarcel return error; 277130803Smarcel} 278130803Smarcel 279130803Smarcel#endif 28098944Sobrien 28198944Sobrien/* 28246283Sdfr * Give next character to user as result of read. 283130803Smarcel */ 28419370Spstint 28519370Spstureadc(c, uio) 28619370Spst register int c; 287130803Smarcel register struct uio *uio; 288130803Smarcel{ 28919370Spst register struct iovec *iov; 290130803Smarcel 291130803Smarcelagain: 29219370Spst if (uio->uio_iovcnt == 0 || uio->uio_resid == 0) 293130803Smarcel panic("ureadc"); 29498944Sobrien iov = uio->uio_iov; 295130803Smarcel if (iov->iov_len == 0) { 29619370Spst uio->uio_iovcnt--; 29798944Sobrien uio->uio_iov++; 29898944Sobrien goto again; 29998944Sobrien } 30098944Sobrien switch (uio->uio_segflg) { 301130803Smarcel 302130803Smarcel case UIO_USERSPACE: 30319370Spst if (subyte(iov->iov_base, c) < 0) 30419370Spst return (EFAULT); 30519370Spst break; 30619370Spst 30719370Spst case UIO_SYSSPACE: 30819370Spst *iov->iov_base = c; 30919370Spst break; 31098944Sobrien 31198944Sobrien case UIO_USERISPACE: 31219370Spst if (suibyte(iov->iov_base, c) < 0) 313130803Smarcel return (EFAULT); 31419370Spst break; 315130803Smarcel case UIO_NOCOPY: 31619370Spst break; 31719370Spst } 31819370Spst iov->iov_base++; 319130803Smarcel iov->iov_len--; 32019370Spst uio->uio_resid--; 32198944Sobrien uio->uio_offset++; 32219370Spst return (0); 32319370Spst} 32498944Sobrien 32598944Sobrien/* 32619370Spst * General routine to allocate a hash table. 32798944Sobrien */ 32819370Spstvoid * 329130803Smarcelhashinit(elements, type, hashmask) 33019370Spst int elements; 33119370Spst struct malloc_type *type; 33246283Sdfr u_long *hashmask; 33398944Sobrien{ 33419370Spst long hashsize; 33598944Sobrien LIST_HEAD(generic, generic) *hashtbl; 33619370Spst int i; 33719370Spst 33819370Spst if (elements <= 0) 33919370Spst panic("hashinit: bad elements"); 34019370Spst for (hashsize = 1; hashsize <= elements; hashsize <<= 1) 34119370Spst continue; 34298944Sobrien hashsize >>= 1; 34319370Spst hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), type, M_WAITOK); 34419370Spst for (i = 0; i < hashsize; i++) 34519370Spst LIST_INIT(&hashtbl[i]); 34619370Spst *hashmask = hashsize - 1; 34798944Sobrien return (hashtbl); 348130803Smarcel} 349130803Smarcel 350130803Smarcelstatic int primes[] = { 1, 13, 31, 61, 127, 251, 509, 761, 1021, 1531, 2039, 351130803Smarcel 2557, 3067, 3583, 4093, 4603, 5119, 5623, 6143, 6653, 352130803Smarcel 7159, 7673, 8191, 12281, 16381, 24571, 32749 }; 353130803Smarcel#define NPRIMES (sizeof(primes) / sizeof(primes[0])) 354130803Smarcel 355130803Smarcel/* 356130803Smarcel * General routine to allocate a prime number sized hash table. 35719370Spst */ 35819370Spstvoid * 35919370Spstphashinit(elements, type, nentries) 36019370Spst int elements; 36119370Spst struct malloc_type *type; 36219370Spst u_long *nentries; 36319370Spst{ 36419370Spst long hashsize; 36519370Spst LIST_HEAD(generic, generic) *hashtbl; 366130803Smarcel int i; 36719370Spst 36819370Spst if (elements <= 0) 36919370Spst panic("phashinit: bad elements"); 37019370Spst for (i = 1, hashsize = primes[1]; hashsize <= elements;) { 37119370Spst i++; 37246283Sdfr if (i == NPRIMES) 37319370Spst break; 37419370Spst hashsize = primes[i]; 37519370Spst } 37619370Spst hashsize = primes[i - 1]; 37798944Sobrien hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), type, M_WAITOK); 37898944Sobrien for (i = 0; i < hashsize; i++) 37919370Spst LIST_INIT(&hashtbl[i]); 38019370Spst *nentries = hashsize; 38119370Spst return (hashtbl); 38219370Spst} 38319370Spst 38419370Spststatic void 38519370Spstuio_yield() 38619370Spst{ 38719370Spst struct thread *td; 38819370Spst 38919370Spst td = curthread; 39019370Spst mtx_lock_spin(&sched_lock); 39119370Spst DROP_GIANT_NOSWITCH(); 39219370Spst td->td_ksegrp->kg_pri.pri_level = td->td_ksegrp->kg_pri.pri_user; 393130803Smarcel setrunqueue(td); 39419370Spst td->td_proc->p_stats->p_ru.ru_nivcsw++; 39598944Sobrien mi_switch(); 396130803Smarcel mtx_unlock_spin(&sched_lock); 39719370Spst PICKUP_GIANT(); 398130803Smarcel} 39919370Spst 40019370Spstint 401130803Smarcelcopyinfrom(const void *src, void *dst, size_t len, int seg) 40219370Spst{ 40319370Spst int error = 0; 40419370Spst 40546283Sdfr switch (seg) { 40646283Sdfr case UIO_USERSPACE: 40719370Spst error = copyin(src, dst, len); 40819370Spst break; 409130803Smarcel case UIO_SYSSPACE: 410130803Smarcel bcopy(src, dst, len); 411130803Smarcel break; 41298944Sobrien default: 413130803Smarcel panic("copyinfrom: bad seg %d\n", seg); 41419370Spst } 41519370Spst return (error); 41619370Spst} 41719370Spst 41819370Spstint 41919370Spstcopyinstrfrom(const void *src, void *dst, size_t len, size_t *copied, int seg) 42019370Spst{ 42119370Spst int error = 0; 42219370Spst 42319370Spst switch (seg) { 42419370Spst case UIO_USERSPACE: 42519370Spst error = copyinstr(src, dst, len, copied); 42646283Sdfr break; 42798944Sobrien case UIO_SYSSPACE: 42898944Sobrien error = copystr(src, dst, len, copied); 42998944Sobrien break; 43098944Sobrien default: 43198944Sobrien panic("copyinstrfrom: bad seg %d\n", seg); 43298944Sobrien } 43346283Sdfr return (error); 43498944Sobrien} 43598944Sobrien