subr_uio.c revision 111737
11541Srgrimes/*
21541Srgrimes * Copyright (c) 1982, 1986, 1991, 1993
31541Srgrimes *	The Regents of the University of California.  All rights reserved.
41541Srgrimes * (c) UNIX System Laboratories, Inc.
51541Srgrimes * All or some portions of this file are derived from material licensed
61541Srgrimes * to the University of California by American Telephone and Telegraph
71541Srgrimes * Co. or Unix System Laboratories, Inc. and are reproduced herein with
81541Srgrimes * the permission of UNIX System Laboratories, Inc.
91541Srgrimes *
101541Srgrimes * Redistribution and use in source and binary forms, with or without
111541Srgrimes * modification, are permitted provided that the following conditions
121541Srgrimes * are met:
131541Srgrimes * 1. Redistributions of source code must retain the above copyright
141541Srgrimes *    notice, this list of conditions and the following disclaimer.
151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright
161541Srgrimes *    notice, this list of conditions and the following disclaimer in the
171541Srgrimes *    documentation and/or other materials provided with the distribution.
181541Srgrimes * 3. All advertising materials mentioning features or use of this software
191541Srgrimes *    must display the following acknowledgement:
201541Srgrimes *	This product includes software developed by the University of
211541Srgrimes *	California, Berkeley and its contributors.
221541Srgrimes * 4. Neither the name of the University nor the names of its contributors
231541Srgrimes *    may be used to endorse or promote products derived from this software
241541Srgrimes *    without specific prior written permission.
251541Srgrimes *
261541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
271541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
281541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
291541Srgrimes * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
301541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
311541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
321541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
331541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
341541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
351541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
361541Srgrimes * SUCH DAMAGE.
371541Srgrimes *
381541Srgrimes *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
3950477Speter * $FreeBSD: head/sys/kern/kern_subr.c 111737 2003-03-02 15:08:33Z des $
401541Srgrimes */
411541Srgrimes
4298849Sken#include "opt_zero.h"
4398849Sken
441541Srgrimes#include <sys/param.h>
451541Srgrimes#include <sys/systm.h>
4644218Sbde#include <sys/kernel.h>
4765557Sjasone#include <sys/ktr.h>
4876166Smarkm#include <sys/lock.h>
4976166Smarkm#include <sys/mutex.h>
501541Srgrimes#include <sys/proc.h>
511541Srgrimes#include <sys/malloc.h>
5243529Sbde#include <sys/resourcevar.h>
53104964Sjeff#include <sys/sched.h>
5478431Swollman#include <sys/sysctl.h>
5532702Sdyson#include <sys/vnode.h>
561541Srgrimes
5731853Sdyson#include <vm/vm.h>
5831853Sdyson#include <vm/vm_page.h>
5931853Sdyson#include <vm/vm_map.h>
6099848Sken#ifdef ZERO_COPY_SOCKETS
6199848Sken#include <vm/vm_param.h>
6299848Sken#endif
6399848Sken#if defined(ZERO_COPY_SOCKETS) || defined(ENABLE_VFS_IOOPT)
6499848Sken#include <vm/vm_object.h>
6599848Sken#endif
6631853Sdyson
67111737SdesSYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
6878431Swollman	"Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
6978431Swollman
7098849Sken#ifdef ZERO_COPY_SOCKETS
7198849Sken/* Declared in uipc_socket.c */
7298849Skenextern int so_zero_copy_receive;
7398849Sken
7498849Skenstatic int
75111737Sdesvm_pgmoveco(vm_map_t mapa, vm_object_t srcobj, vm_offset_t kaddr,
76111737Sdes    vm_offset_t uaddr)
7798849Sken{
7898849Sken	vm_map_t map = mapa;
7998849Sken	vm_page_t kern_pg, user_pg;
8098849Sken	vm_object_t uobject;
8198849Sken	vm_map_entry_t entry;
8298849Sken	vm_pindex_t upindex, kpindex;
8398849Sken	vm_prot_t prot;
8498849Sken	boolean_t wired;
8598849Sken
8698849Sken	/*
8798849Sken	 * First lookup the kernel page.
8898849Sken	 */
8998849Sken	kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
9098849Sken
9198849Sken	if ((vm_map_lookup(&map, uaddr,
9298849Sken			   VM_PROT_READ, &entry, &uobject,
9398849Sken			   &upindex, &prot, &wired)) != KERN_SUCCESS) {
9498849Sken		return(EFAULT);
9598849Sken	}
9698849Sken	if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
97107371Salc		do
98107371Salc			vm_page_lock_queues();
99107371Salc		while (vm_page_sleep_if_busy(user_pg, 1, "vm_pgmoveco"));
10098849Sken		vm_page_busy(user_pg);
101107371Salc		pmap_remove_all(user_pg);
10298849Sken		vm_page_free(user_pg);
103108139Salc	} else
104108139Salc		vm_page_lock_queues();
10598849Sken	if (kern_pg->busy || ((kern_pg->queue - kern_pg->pc) == PQ_FREE) ||
10698849Sken	    (kern_pg->hold_count != 0)|| (kern_pg->flags & PG_BUSY)) {
10798849Sken		printf("vm_pgmoveco: pindex(%lu), busy(%d), PG_BUSY(%d), "
10898849Sken		       "hold(%d) paddr(0x%lx)\n", (u_long)kern_pg->pindex,
10998849Sken			kern_pg->busy, (kern_pg->flags & PG_BUSY) ? 1 : 0,
11098849Sken			kern_pg->hold_count, (u_long)kern_pg->phys_addr);
11198849Sken		if ((kern_pg->queue - kern_pg->pc) == PQ_FREE)
11298849Sken			panic("vm_pgmoveco: renaming free page");
11398849Sken		else
11498849Sken			panic("vm_pgmoveco: renaming busy page");
11598849Sken	}
11698849Sken	kpindex = kern_pg->pindex;
11798849Sken	vm_page_busy(kern_pg);
11898849Sken	vm_page_rename(kern_pg, uobject, upindex);
11998849Sken	vm_page_flag_clear(kern_pg, PG_BUSY);
12098849Sken	kern_pg->valid = VM_PAGE_BITS_ALL;
121108139Salc	vm_page_unlock_queues();
122111737Sdes
12398849Sken	vm_map_lookup_done(map, entry);
12498849Sken	return(KERN_SUCCESS);
12598849Sken}
12698849Sken#endif /* ZERO_COPY_SOCKETS */
12798849Sken
1281549Srgrimesint
129111737Sdesuiomove(caddr_t cp, int n, struct uio *uio)
1301541Srgrimes{
13183366Sjulian	struct thread *td = curthread;
132111737Sdes	struct iovec *iov;
1331541Srgrimes	u_int cnt;
13444681Sjulian	int error = 0;
13544681Sjulian	int save = 0;
1361541Srgrimes
13742408Seivind	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
13842453Seivind	    ("uiomove: mode"));
13983366Sjulian	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
14042453Seivind	    ("uiomove proc"));
14142408Seivind
14283366Sjulian	if (td) {
14383420Sjhb		mtx_lock_spin(&sched_lock);
14483366Sjulian		save = td->td_flags & TDF_DEADLKTREAT;
14583366Sjulian		td->td_flags |= TDF_DEADLKTREAT;
14683420Sjhb		mtx_unlock_spin(&sched_lock);
14744681Sjulian	}
14844681Sjulian
1491541Srgrimes	while (n > 0 && uio->uio_resid) {
1501541Srgrimes		iov = uio->uio_iov;
1511541Srgrimes		cnt = iov->iov_len;
1521541Srgrimes		if (cnt == 0) {
1531541Srgrimes			uio->uio_iov++;
1541541Srgrimes			uio->uio_iovcnt--;
1551541Srgrimes			continue;
1561541Srgrimes		}
1571541Srgrimes		if (cnt > n)
1581541Srgrimes			cnt = n;
1596324Sdg
1601541Srgrimes		switch (uio->uio_segflg) {
1611541Srgrimes
1621541Srgrimes		case UIO_USERSPACE:
16370861Sjake			if (ticks - PCPU_GET(switchticks) >= hogticks)
16443529Sbde				uio_yield();
1651541Srgrimes			if (uio->uio_rw == UIO_READ)
1661541Srgrimes				error = copyout(cp, iov->iov_base, cnt);
1671541Srgrimes			else
1681541Srgrimes				error = copyin(iov->iov_base, cp, cnt);
1691541Srgrimes			if (error)
17090413Stmm				goto out;
1711541Srgrimes			break;
1721541Srgrimes
1731541Srgrimes		case UIO_SYSSPACE:
1741541Srgrimes			if (uio->uio_rw == UIO_READ)
17598998Salfred				bcopy(cp, iov->iov_base, cnt);
1761541Srgrimes			else
17798998Salfred				bcopy(iov->iov_base, cp, cnt);
1781541Srgrimes			break;
1797611Sdg		case UIO_NOCOPY:
1807611Sdg			break;
1811541Srgrimes		}
182104908Smike		iov->iov_base = (char *)iov->iov_base + cnt;
1831541Srgrimes		iov->iov_len -= cnt;
1841541Srgrimes		uio->uio_resid -= cnt;
1851541Srgrimes		uio->uio_offset += cnt;
1861541Srgrimes		cp += cnt;
1871541Srgrimes		n -= cnt;
1881541Srgrimes	}
18990413Stmmout:
19083366Sjulian	if (td != curthread) printf("uiomove: IT CHANGED!");
19183366Sjulian	td = curthread;	/* Might things have changed in copyin/copyout? */
19283420Sjhb	if (td) {
19383420Sjhb		mtx_lock_spin(&sched_lock);
19483366Sjulian		td->td_flags = (td->td_flags & ~TDF_DEADLKTREAT) | save;
19583420Sjhb		mtx_unlock_spin(&sched_lock);
19683420Sjhb	}
19744681Sjulian	return (error);
1981541Srgrimes}
1991541Srgrimes
20098849Sken#if defined(ENABLE_VFS_IOOPT) || defined(ZERO_COPY_SOCKETS)
20196080Salc/*
20296080Salc * Experimental support for zero-copy I/O
20396080Salc */
20498849Skenstatic int
205111737Sdesuserspaceco(caddr_t cp, u_int cnt, struct uio *uio, struct vm_object *obj,
206111737Sdes    int disposable)
20798849Sken{
20898849Sken	struct iovec *iov;
20998849Sken	int error;
21098849Sken
21198849Sken	iov = uio->uio_iov;
21298849Sken
21398849Sken#ifdef ZERO_COPY_SOCKETS
21498849Sken
21598849Sken	if (uio->uio_rw == UIO_READ) {
21698849Sken		if ((so_zero_copy_receive != 0)
21798849Sken		 && (obj != NULL)
21898849Sken		 && ((cnt & PAGE_MASK) == 0)
21998849Sken		 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
22098849Sken		 && ((uio->uio_offset & PAGE_MASK) == 0)
22198849Sken		 && ((((intptr_t) cp) & PAGE_MASK) == 0)
22298849Sken		 && (obj->type == OBJT_DEFAULT)
22398849Sken		 && (disposable != 0)) {
22498849Sken			/* SOCKET: use page-trading */
22598849Sken			/*
22698849Sken			 * We only want to call vm_pgmoveco() on
22798849Sken			 * disposeable pages, since it gives the
22898849Sken			 * kernel page to the userland process.
22998849Sken			 */
23098849Sken			error =	vm_pgmoveco(&curproc->p_vmspace->vm_map,
231111737Sdes					    obj, (vm_offset_t)cp,
23298849Sken					    (vm_offset_t)iov->iov_base);
23398849Sken
23498849Sken			/*
23598849Sken			 * If we get an error back, attempt
23698849Sken			 * to use copyout() instead.  The
23798849Sken			 * disposable page should be freed
23898849Sken			 * automatically if we weren't able to move
23998849Sken			 * it into userland.
24098849Sken			 */
24198849Sken			if (error != 0)
24298849Sken				error = copyout(cp, iov->iov_base, cnt);
24398849Sken#ifdef ENABLE_VFS_IOOPT
24498849Sken		} else if ((vfs_ioopt != 0)
24598849Sken		 && ((cnt & PAGE_MASK) == 0)
24698849Sken		 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
24798849Sken		 && ((uio->uio_offset & PAGE_MASK) == 0)
24898849Sken		 && ((((intptr_t) cp) & PAGE_MASK) == 0)) {
24998849Sken			error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
25098849Sken					   uio->uio_offset, cnt,
25198849Sken					   (vm_offset_t) iov->iov_base, NULL);
25298849Sken#endif /* ENABLE_VFS_IOOPT */
25398849Sken		} else {
25498849Sken			error = copyout(cp, iov->iov_base, cnt);
25598849Sken		}
25698849Sken	} else {
25798849Sken		error = copyin(iov->iov_base, cp, cnt);
25898849Sken	}
25998849Sken#else /* ZERO_COPY_SOCKETS */
26098849Sken	if (uio->uio_rw == UIO_READ) {
26198849Sken#ifdef ENABLE_VFS_IOOPT
26298849Sken		if ((vfs_ioopt != 0)
26398849Sken		 && ((cnt & PAGE_MASK) == 0)
26498849Sken		 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
26598849Sken		 && ((uio->uio_offset & PAGE_MASK) == 0)
26698849Sken		 && ((((intptr_t) cp) & PAGE_MASK) == 0)) {
26798849Sken			error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
26898849Sken					   uio->uio_offset, cnt,
26998849Sken					   (vm_offset_t) iov->iov_base, NULL);
27098849Sken		} else
27198849Sken#endif /* ENABLE_VFS_IOOPT */
27298849Sken		{
27398849Sken			error = copyout(cp, iov->iov_base, cnt);
27498849Sken		}
27598849Sken	} else {
27698849Sken		error = copyin(iov->iov_base, cp, cnt);
27798849Sken	}
27898849Sken#endif /* ZERO_COPY_SOCKETS */
27998849Sken
28098849Sken	return (error);
28198849Sken}
28298849Sken
28331853Sdysonint
284111737Sdesuiomoveco(caddr_t cp, int n, struct uio *uio, struct vm_object *obj,
285111737Sdes    int disposable)
28631853Sdyson{
28731853Sdyson	struct iovec *iov;
28831853Sdyson	u_int cnt;
28931853Sdyson	int error;
29031853Sdyson
29142408Seivind	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
29242453Seivind	    ("uiomoveco: mode"));
29383366Sjulian	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
29442453Seivind	    ("uiomoveco proc"));
29542408Seivind
29631853Sdyson	while (n > 0 && uio->uio_resid) {
29731853Sdyson		iov = uio->uio_iov;
29831853Sdyson		cnt = iov->iov_len;
29931853Sdyson		if (cnt == 0) {
30031853Sdyson			uio->uio_iov++;
30131853Sdyson			uio->uio_iovcnt--;
30231853Sdyson			continue;
30331853Sdyson		}
30431853Sdyson		if (cnt > n)
30531853Sdyson			cnt = n;
30631853Sdyson
30731853Sdyson		switch (uio->uio_segflg) {
30831853Sdyson
30931853Sdyson		case UIO_USERSPACE:
31070861Sjake			if (ticks - PCPU_GET(switchticks) >= hogticks)
31143529Sbde				uio_yield();
31298849Sken
31398849Sken			error = userspaceco(cp, cnt, uio, obj, disposable);
31498849Sken
31531853Sdyson			if (error)
31631853Sdyson				return (error);
31731853Sdyson			break;
31831853Sdyson
31931853Sdyson		case UIO_SYSSPACE:
32031853Sdyson			if (uio->uio_rw == UIO_READ)
32198998Salfred				bcopy(cp, iov->iov_base, cnt);
32231853Sdyson			else
32398998Salfred				bcopy(iov->iov_base, cp, cnt);
32431853Sdyson			break;
32531853Sdyson		case UIO_NOCOPY:
32631853Sdyson			break;
32731853Sdyson		}
328104908Smike		iov->iov_base = (char *)iov->iov_base + cnt;
32931853Sdyson		iov->iov_len -= cnt;
33031853Sdyson		uio->uio_resid -= cnt;
33131853Sdyson		uio->uio_offset += cnt;
33231853Sdyson		cp += cnt;
33331853Sdyson		n -= cnt;
33431853Sdyson	}
33531853Sdyson	return (0);
33631853Sdyson}
33798849Sken#endif /* ENABLE_VFS_IOOPT || ZERO_COPY_SOCKETS */
33831853Sdyson
33998849Sken#ifdef ENABLE_VFS_IOOPT
34098849Sken
34196080Salc/*
34296080Salc * Experimental support for zero-copy I/O
34396080Salc */
34432286Sdysonint
345111737Sdesuioread(int n, struct uio *uio, struct vm_object *obj, int *nread)
34632286Sdyson{
34732286Sdyson	int npagesmoved;
34832286Sdyson	struct iovec *iov;
34932286Sdyson	u_int cnt, tcnt;
35032286Sdyson	int error;
35132286Sdyson
35232286Sdyson	*nread = 0;
35333109Sdyson	if (vfs_ioopt < 2)
35432702Sdyson		return 0;
35533109Sdyson
35632286Sdyson	error = 0;
35732286Sdyson
35832286Sdyson	while (n > 0 && uio->uio_resid) {
35932286Sdyson		iov = uio->uio_iov;
36032286Sdyson		cnt = iov->iov_len;
36132286Sdyson		if (cnt == 0) {
36232286Sdyson			uio->uio_iov++;
36332286Sdyson			uio->uio_iovcnt--;
36432286Sdyson			continue;
36532286Sdyson		}
36632286Sdyson		if (cnt > n)
36732286Sdyson			cnt = n;
36832286Sdyson
36932286Sdyson		if ((uio->uio_segflg == UIO_USERSPACE) &&
37037649Sbde			((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) &&
37132286Sdyson				 ((uio->uio_offset & PAGE_MASK) == 0) ) {
37232286Sdyson
37332286Sdyson			if (cnt < PAGE_SIZE)
37432286Sdyson				break;
37532286Sdyson
37632286Sdyson			cnt &= ~PAGE_MASK;
37732286Sdyson
37870861Sjake			if (ticks - PCPU_GET(switchticks) >= hogticks)
37943529Sbde				uio_yield();
38032286Sdyson			error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
38132286Sdyson						uio->uio_offset, cnt,
38232286Sdyson						(vm_offset_t) iov->iov_base, &npagesmoved);
38332286Sdyson
38432286Sdyson			if (npagesmoved == 0)
38532286Sdyson				break;
38632286Sdyson
38732286Sdyson			tcnt = npagesmoved * PAGE_SIZE;
38838068Sphk			cnt = tcnt;
38932286Sdyson
39032286Sdyson			if (error)
39132286Sdyson				break;
39232286Sdyson
393104908Smike			iov->iov_base = (char *)iov->iov_base + cnt;
39432286Sdyson			iov->iov_len -= cnt;
39532286Sdyson			uio->uio_resid -= cnt;
39632286Sdyson			uio->uio_offset += cnt;
39732286Sdyson			*nread += cnt;
39832286Sdyson			n -= cnt;
39932286Sdyson		} else {
40032286Sdyson			break;
40132286Sdyson		}
40232286Sdyson	}
40332286Sdyson	return error;
40432286Sdyson}
40598849Sken#endif /* ENABLE_VFS_IOOPT */
40648677Smckusick
4071541Srgrimes/*
4081541Srgrimes * Give next character to user as result of read.
4091541Srgrimes */
4101549Srgrimesint
411111737Sdesureadc(int c, struct uio *uio)
4121541Srgrimes{
413111737Sdes	struct iovec *iov;
414111737Sdes	char *iov_base;
4151541Srgrimes
4161541Srgrimesagain:
4171541Srgrimes	if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
4181541Srgrimes		panic("ureadc");
4191541Srgrimes	iov = uio->uio_iov;
4201541Srgrimes	if (iov->iov_len == 0) {
4211541Srgrimes		uio->uio_iovcnt--;
4221541Srgrimes		uio->uio_iov++;
4231541Srgrimes		goto again;
4241541Srgrimes	}
4251541Srgrimes	switch (uio->uio_segflg) {
4261541Srgrimes
4271541Srgrimes	case UIO_USERSPACE:
4281541Srgrimes		if (subyte(iov->iov_base, c) < 0)
4291541Srgrimes			return (EFAULT);
4301541Srgrimes		break;
4311541Srgrimes
4321541Srgrimes	case UIO_SYSSPACE:
433104908Smike		iov_base = iov->iov_base;
434104908Smike		*iov_base = c;
435104908Smike		iov->iov_base = iov_base;
4361541Srgrimes		break;
4371541Srgrimes
4388177Sdg	case UIO_NOCOPY:
4398177Sdg		break;
4401541Srgrimes	}
441104908Smike	iov->iov_base = (char *)iov->iov_base + 1;
4421541Srgrimes	iov->iov_len--;
4431541Srgrimes	uio->uio_resid--;
4441541Srgrimes	uio->uio_offset++;
4451541Srgrimes	return (0);
4461541Srgrimes}
4471541Srgrimes
4481541Srgrimes/*
4491541Srgrimes * General routine to allocate a hash table.
4501541Srgrimes */
4511541Srgrimesvoid *
452111737Sdeshashinit(int elements, struct malloc_type *type, u_long *hashmask)
4531541Srgrimes{
4541541Srgrimes	long hashsize;
45560938Sjake	LIST_HEAD(generic, generic) *hashtbl;
4561541Srgrimes	int i;
4571541Srgrimes
4581541Srgrimes	if (elements <= 0)
4598364Sdg		panic("hashinit: bad elements");
4601541Srgrimes	for (hashsize = 1; hashsize <= elements; hashsize <<= 1)
4611541Srgrimes		continue;
4621541Srgrimes	hashsize >>= 1;
463111119Simp	hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), type, M_WAITOK);
4641541Srgrimes	for (i = 0; i < hashsize; i++)
4651541Srgrimes		LIST_INIT(&hashtbl[i]);
4661541Srgrimes	*hashmask = hashsize - 1;
4671541Srgrimes	return (hashtbl);
4681541Srgrimes}
4697611Sdg
47099098Siedowsevoid
471111737Sdeshashdestroy(void *vhashtbl, struct malloc_type *type, u_long hashmask)
47299098Siedowse{
47399098Siedowse	LIST_HEAD(generic, generic) *hashtbl, *hp;
47499098Siedowse
47599098Siedowse	hashtbl = vhashtbl;
47699098Siedowse	for (hp = hashtbl; hp <= &hashtbl[hashmask]; hp++)
47799098Siedowse		if (!LIST_EMPTY(hp))
47899098Siedowse			panic("hashdestroy: hash not empty");
47999098Siedowse	free(hashtbl, type);
48099098Siedowse}
48199098Siedowse
4827683Sdgstatic int primes[] = { 1, 13, 31, 61, 127, 251, 509, 761, 1021, 1531, 2039,
4837683Sdg			2557, 3067, 3583, 4093, 4603, 5119, 5623, 6143, 6653,
4847611Sdg			7159, 7673, 8191, 12281, 16381, 24571, 32749 };
48526205Salex#define NPRIMES (sizeof(primes) / sizeof(primes[0]))
4867611Sdg
4877611Sdg/*
4887611Sdg * General routine to allocate a prime number sized hash table.
4897611Sdg */
4907611Sdgvoid *
491111737Sdesphashinit(int elements, struct malloc_type *type, u_long *nentries)
4927611Sdg{
4937611Sdg	long hashsize;
49460938Sjake	LIST_HEAD(generic, generic) *hashtbl;
4957611Sdg	int i;
4967611Sdg
4977611Sdg	if (elements <= 0)
4988364Sdg		panic("phashinit: bad elements");
4997611Sdg	for (i = 1, hashsize = primes[1]; hashsize <= elements;) {
5007611Sdg		i++;
5017611Sdg		if (i == NPRIMES)
5027611Sdg			break;
5037611Sdg		hashsize = primes[i];
5047611Sdg	}
5057611Sdg	hashsize = primes[i - 1];
506111119Simp	hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), type, M_WAITOK);
5077611Sdg	for (i = 0; i < hashsize; i++)
5087611Sdg		LIST_INIT(&hashtbl[i]);
5097611Sdg	*nentries = hashsize;
5107611Sdg	return (hashtbl);
5117611Sdg}
51243529Sbde
51383959Sdillonvoid
514111737Sdesuio_yield(void)
51543529Sbde{
51683366Sjulian	struct thread *td;
51743529Sbde
51883366Sjulian	td = curthread;
51972200Sbmilekic	mtx_lock_spin(&sched_lock);
52088900Sjhb	DROP_GIANT();
521104964Sjeff	sched_prio(td, td->td_ksegrp->kg_user_pri); /* XXXKSE */
52283366Sjulian	td->td_proc->p_stats->p_ru.ru_nivcsw++;
52343529Sbde	mi_switch();
52472200Sbmilekic	mtx_unlock_spin(&sched_lock);
52568808Sjhb	PICKUP_GIANT();
52643529Sbde}
52772537Sjlemon
52872537Sjlemonint
52972537Sjlemoncopyinfrom(const void *src, void *dst, size_t len, int seg)
53072537Sjlemon{
53172537Sjlemon	int error = 0;
53272537Sjlemon
53372537Sjlemon	switch (seg) {
53472537Sjlemon	case UIO_USERSPACE:
53572537Sjlemon		error = copyin(src, dst, len);
53672537Sjlemon		break;
53772537Sjlemon	case UIO_SYSSPACE:
53872537Sjlemon		bcopy(src, dst, len);
53972537Sjlemon		break;
54072537Sjlemon	default:
54172537Sjlemon		panic("copyinfrom: bad seg %d\n", seg);
54272537Sjlemon	}
54372537Sjlemon	return (error);
54472537Sjlemon}
54572537Sjlemon
54672537Sjlemonint
54772537Sjlemoncopyinstrfrom(const void *src, void *dst, size_t len, size_t *copied, int seg)
54872537Sjlemon{
54972537Sjlemon	int error = 0;
55072537Sjlemon
55172537Sjlemon	switch (seg) {
55272537Sjlemon	case UIO_USERSPACE:
55372537Sjlemon		error = copyinstr(src, dst, len, copied);
55472537Sjlemon		break;
55572537Sjlemon	case UIO_SYSSPACE:
55672537Sjlemon		error = copystr(src, dst, len, copied);
55772537Sjlemon		break;
55872537Sjlemon	default:
55972537Sjlemon		panic("copyinstrfrom: bad seg %d\n", seg);
56072537Sjlemon	}
56172537Sjlemon	return (error);
56272537Sjlemon}
563