Deleted Added
sdiff udiff text old ( 98477 ) new ( 98849 )
full compact
subr_uio.c (98477) subr_uio.c (98849)
1/*
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 22 unchanged lines hidden (view full) ---

31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
1/*
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 22 unchanged lines hidden (view full) ---

31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
39 * $FreeBSD: head/sys/kern/kern_subr.c 98477 2002-06-20 07:08:43Z peter $
39 * $FreeBSD: head/sys/kern/kern_subr.c 98849 2002-06-26 03:37:47Z ken $
40 */
41
40 */
41
42#include "opt_zero.h"
43
42#include <sys/param.h>
43#include <sys/systm.h>
44#include <sys/kernel.h>
45#include <sys/ktr.h>
46#include <sys/lock.h>
47#include <sys/mutex.h>
48#include <sys/proc.h>
49#include <sys/malloc.h>
50#include <sys/resourcevar.h>
51#include <sys/sysctl.h>
52#include <sys/vnode.h>
53
54#include <vm/vm.h>
55#include <vm/vm_page.h>
56#include <vm/vm_map.h>
57
58SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
59 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
60
44#include <sys/param.h>
45#include <sys/systm.h>
46#include <sys/kernel.h>
47#include <sys/ktr.h>
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/proc.h>
51#include <sys/malloc.h>
52#include <sys/resourcevar.h>
53#include <sys/sysctl.h>
54#include <sys/vnode.h>
55
56#include <vm/vm.h>
57#include <vm/vm_page.h>
58#include <vm/vm_map.h>
59
60SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
61 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
62
63#ifdef ZERO_COPY_SOCKETS
64#include <vm/vm.h>
65#include <vm/vm_param.h>
66#include <sys/lock.h>
67#include <vm/pmap.h>
68#include <vm/vm_map.h>
69#include <vm/vm_page.h>
70#include <vm/vm_object.h>
71#include <vm/vm_pager.h>
72#include <vm/vm_kern.h>
73#include <vm/vm_extern.h>
74#include <vm/swap_pager.h>
75#include <sys/mbuf.h>
76#include <machine/cpu.h>
77
78/* Declared in uipc_socket.c */
79extern int so_zero_copy_receive;
80
81static int vm_pgmoveco(vm_map_t mapa, vm_object_t srcobj, vm_offset_t kaddr,
82 vm_offset_t uaddr);
83static int userspaceco(caddr_t cp, u_int cnt, struct uio *uio,
84 struct vm_object *obj, int disposable);
85
86static int
87vm_pgmoveco(mapa, srcobj, kaddr, uaddr)
88 vm_map_t mapa;
89 vm_object_t srcobj;
90 vm_offset_t kaddr, uaddr;
91{
92 vm_map_t map = mapa;
93 vm_page_t kern_pg, user_pg;
94 vm_object_t uobject;
95 vm_map_entry_t entry;
96 vm_pindex_t upindex, kpindex;
97 vm_prot_t prot;
98 boolean_t wired;
99
100 /*
101 * First lookup the kernel page.
102 */
103 kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
104
105 if ((vm_map_lookup(&map, uaddr,
106 VM_PROT_READ, &entry, &uobject,
107 &upindex, &prot, &wired)) != KERN_SUCCESS) {
108 return(EFAULT);
109 }
110 if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
111 vm_page_sleep_busy(user_pg, 1, "vm_pgmoveco");
112 pmap_remove(map->pmap, uaddr, uaddr+PAGE_SIZE);
113 vm_page_busy(user_pg);
114 vm_page_free(user_pg);
115 }
116
117 if (kern_pg->busy || ((kern_pg->queue - kern_pg->pc) == PQ_FREE) ||
118 (kern_pg->hold_count != 0)|| (kern_pg->flags & PG_BUSY)) {
119 printf("vm_pgmoveco: pindex(%lu), busy(%d), PG_BUSY(%d), "
120 "hold(%d) paddr(0x%lx)\n", (u_long)kern_pg->pindex,
121 kern_pg->busy, (kern_pg->flags & PG_BUSY) ? 1 : 0,
122 kern_pg->hold_count, (u_long)kern_pg->phys_addr);
123 if ((kern_pg->queue - kern_pg->pc) == PQ_FREE)
124 panic("vm_pgmoveco: renaming free page");
125 else
126 panic("vm_pgmoveco: renaming busy page");
127 }
128 kpindex = kern_pg->pindex;
129 vm_page_busy(kern_pg);
130 vm_page_rename(kern_pg, uobject, upindex);
131 vm_page_flag_clear(kern_pg, PG_BUSY);
132 kern_pg->valid = VM_PAGE_BITS_ALL;
133
134 vm_map_lookup_done(map, entry);
135 return(KERN_SUCCESS);
136}
137#endif /* ZERO_COPY_SOCKETS */
138
61int
62uiomove(cp, n, uio)
63 register caddr_t cp;
64 register int n;
65 register struct uio *uio;
66{
67 struct thread *td = curthread;
68 register struct iovec *iov;

--- 59 unchanged lines hidden (view full) ---

128 if (td) {
129 mtx_lock_spin(&sched_lock);
130 td->td_flags = (td->td_flags & ~TDF_DEADLKTREAT) | save;
131 mtx_unlock_spin(&sched_lock);
132 }
133 return (error);
134}
135
139int
140uiomove(cp, n, uio)
141 register caddr_t cp;
142 register int n;
143 register struct uio *uio;
144{
145 struct thread *td = curthread;
146 register struct iovec *iov;

--- 59 unchanged lines hidden (view full) ---

206 if (td) {
207 mtx_lock_spin(&sched_lock);
208 td->td_flags = (td->td_flags & ~TDF_DEADLKTREAT) | save;
209 mtx_unlock_spin(&sched_lock);
210 }
211 return (error);
212}
213
136#ifdef ENABLE_VFS_IOOPT
214#if defined(ENABLE_VFS_IOOPT) || defined(ZERO_COPY_SOCKETS)
137/*
138 * Experimental support for zero-copy I/O
139 */
215/*
216 * Experimental support for zero-copy I/O
217 */
218static int
219userspaceco(cp, cnt, uio, obj, disposable)
220 caddr_t cp;
221 u_int cnt;
222 struct uio *uio;
223 struct vm_object *obj;
224 int disposable;
225{
226 struct iovec *iov;
227 int error;
228
229 iov = uio->uio_iov;
230
231#ifdef ZERO_COPY_SOCKETS
232
233 if (uio->uio_rw == UIO_READ) {
234 if ((so_zero_copy_receive != 0)
235 && (obj != NULL)
236 && ((cnt & PAGE_MASK) == 0)
237 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
238 && ((uio->uio_offset & PAGE_MASK) == 0)
239 && ((((intptr_t) cp) & PAGE_MASK) == 0)
240 && (obj->type == OBJT_DEFAULT)
241 && (disposable != 0)) {
242 /* SOCKET: use page-trading */
243 /*
244 * We only want to call vm_pgmoveco() on
245 * disposeable pages, since it gives the
246 * kernel page to the userland process.
247 */
248 error = vm_pgmoveco(&curproc->p_vmspace->vm_map,
249 obj, (vm_offset_t)cp,
250 (vm_offset_t)iov->iov_base);
251
252 /*
253 * If we get an error back, attempt
254 * to use copyout() instead. The
255 * disposable page should be freed
256 * automatically if we weren't able to move
257 * it into userland.
258 */
259 if (error != 0)
260 error = copyout(cp, iov->iov_base, cnt);
261#ifdef ENABLE_VFS_IOOPT
262 } else if ((vfs_ioopt != 0)
263 && ((cnt & PAGE_MASK) == 0)
264 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
265 && ((uio->uio_offset & PAGE_MASK) == 0)
266 && ((((intptr_t) cp) & PAGE_MASK) == 0)) {
267 error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
268 uio->uio_offset, cnt,
269 (vm_offset_t) iov->iov_base, NULL);
270#endif /* ENABLE_VFS_IOOPT */
271 } else {
272 error = copyout(cp, iov->iov_base, cnt);
273 }
274 } else {
275 error = copyin(iov->iov_base, cp, cnt);
276 }
277#else /* ZERO_COPY_SOCKETS */
278 if (uio->uio_rw == UIO_READ) {
279#ifdef ENABLE_VFS_IOOPT
280 if ((vfs_ioopt != 0)
281 && ((cnt & PAGE_MASK) == 0)
282 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
283 && ((uio->uio_offset & PAGE_MASK) == 0)
284 && ((((intptr_t) cp) & PAGE_MASK) == 0)) {
285 error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
286 uio->uio_offset, cnt,
287 (vm_offset_t) iov->iov_base, NULL);
288 } else
289#endif /* ENABLE_VFS_IOOPT */
290 {
291 error = copyout(cp, iov->iov_base, cnt);
292 }
293 } else {
294 error = copyin(iov->iov_base, cp, cnt);
295 }
296#endif /* ZERO_COPY_SOCKETS */
297
298 return (error);
299}
300
140int
301int
141uiomoveco(cp, n, uio, obj)
302uiomoveco(cp, n, uio, obj, disposable)
142 caddr_t cp;
143 int n;
144 struct uio *uio;
145 struct vm_object *obj;
303 caddr_t cp;
304 int n;
305 struct uio *uio;
306 struct vm_object *obj;
307 int disposable;
146{
147 struct iovec *iov;
148 u_int cnt;
149 int error;
150
151 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
152 ("uiomoveco: mode"));
153 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,

--- 10 unchanged lines hidden (view full) ---

164 if (cnt > n)
165 cnt = n;
166
167 switch (uio->uio_segflg) {
168
169 case UIO_USERSPACE:
170 if (ticks - PCPU_GET(switchticks) >= hogticks)
171 uio_yield();
308{
309 struct iovec *iov;
310 u_int cnt;
311 int error;
312
313 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
314 ("uiomoveco: mode"));
315 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,

--- 10 unchanged lines hidden (view full) ---

326 if (cnt > n)
327 cnt = n;
328
329 switch (uio->uio_segflg) {
330
331 case UIO_USERSPACE:
332 if (ticks - PCPU_GET(switchticks) >= hogticks)
333 uio_yield();
172 if (uio->uio_rw == UIO_READ) {
173#ifdef ENABLE_VFS_IOOPT
174 if (vfs_ioopt && ((cnt & PAGE_MASK) == 0) &&
175 ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0) &&
176 ((uio->uio_offset & PAGE_MASK) == 0) &&
177 ((((intptr_t) cp) & PAGE_MASK) == 0)) {
178 error = vm_uiomove(&curproc->p_vmspace->vm_map, obj,
179 uio->uio_offset, cnt,
180 (vm_offset_t) iov->iov_base, NULL);
181 } else
182#endif
183 {
184 error = copyout(cp, iov->iov_base, cnt);
185 }
186 } else {
187 error = copyin(iov->iov_base, cp, cnt);
188 }
334
335 error = userspaceco(cp, cnt, uio, obj, disposable);
336
189 if (error)
190 return (error);
191 break;
192
193 case UIO_SYSSPACE:
194 if (uio->uio_rw == UIO_READ)
195 bcopy((caddr_t)cp, iov->iov_base, cnt);
196 else

--- 6 unchanged lines hidden (view full) ---

203 iov->iov_len -= cnt;
204 uio->uio_resid -= cnt;
205 uio->uio_offset += cnt;
206 cp += cnt;
207 n -= cnt;
208 }
209 return (0);
210}
337 if (error)
338 return (error);
339 break;
340
341 case UIO_SYSSPACE:
342 if (uio->uio_rw == UIO_READ)
343 bcopy((caddr_t)cp, iov->iov_base, cnt);
344 else

--- 6 unchanged lines hidden (view full) ---

351 iov->iov_len -= cnt;
352 uio->uio_resid -= cnt;
353 uio->uio_offset += cnt;
354 cp += cnt;
355 n -= cnt;
356 }
357 return (0);
358}
359#endif /* ENABLE_VFS_IOOPT || ZERO_COPY_SOCKETS */
211
360
361#ifdef ENABLE_VFS_IOOPT
362
212/*
213 * Experimental support for zero-copy I/O
214 */
215int
216uioread(n, uio, obj, nread)
217 int n;
218 struct uio *uio;
219 struct vm_object *obj;

--- 52 unchanged lines hidden (view full) ---

272 *nread += cnt;
273 n -= cnt;
274 } else {
275 break;
276 }
277 }
278 return error;
279}
363/*
364 * Experimental support for zero-copy I/O
365 */
366int
367uioread(n, uio, obj, nread)
368 int n;
369 struct uio *uio;
370 struct vm_object *obj;

--- 52 unchanged lines hidden (view full) ---

423 *nread += cnt;
424 n -= cnt;
425 } else {
426 break;
427 }
428 }
429 return error;
430}
280#endif
431#endif /* ENABLE_VFS_IOOPT */
281
282/*
283 * Give next character to user as result of read.
284 */
285int
286ureadc(c, uio)
287 register int c;
288 register struct uio *uio;

--- 143 unchanged lines hidden --- 		432
433/*
434 * Give next character to user as result of read.
435 */
436int
437ureadc(c, uio)
438 register int c;
439 register struct uio *uio;

--- 143 unchanged lines hidden ---