Deleted Added
sdiff udiff text old ( 254141 ) new ( 255608 )
full compact
subr_uio.c (254141) subr_uio.c (255608)
1/*-
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 21 unchanged lines hidden (view full) ---

30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
35 */
36
37#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 21 unchanged lines hidden (view full) ---

30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
35 */
36
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD: head/sys/kern/subr_uio.c 254141 2013-08-09 11:28:55Z attilio $");
38__FBSDID("$FreeBSD: head/sys/kern/subr_uio.c 255608 2013-09-16 06:25:54Z kib $");
39
39
40#include "opt_zero.h"
41
42#include <sys/param.h>
43#include <sys/systm.h>
44#include <sys/kernel.h>
45#include <sys/limits.h>
46#include <sys/lock.h>
47#include <sys/mman.h>
48#include <sys/proc.h>
49#include <sys/resourcevar.h>
50#include <sys/rwlock.h>
51#include <sys/sched.h>
52#include <sys/sysctl.h>
53#include <sys/vnode.h>
54
55#include <vm/vm.h>
56#include <vm/vm_param.h>
57#include <vm/vm_extern.h>
58#include <vm/vm_page.h>
59#include <vm/vm_pageout.h>
60#include <vm/vm_map.h>
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/limits.h>
44#include <sys/lock.h>
45#include <sys/mman.h>
46#include <sys/proc.h>
47#include <sys/resourcevar.h>
48#include <sys/rwlock.h>
49#include <sys/sched.h>
50#include <sys/sysctl.h>
51#include <sys/vnode.h>
52
53#include <vm/vm.h>
54#include <vm/vm_param.h>
55#include <vm/vm_extern.h>
56#include <vm/vm_page.h>
57#include <vm/vm_pageout.h>
58#include <vm/vm_map.h>
61#ifdef SOCKET_SEND_COW
62#include <vm/vm_object.h>
63#endif
64
65SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
66 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
67
68static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
69
59
60SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, NULL, UIO_MAXIOV,
61 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
62
63static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
64
70#ifdef SOCKET_SEND_COW
71/* Declared in uipc_socket.c */
72extern int so_zero_copy_receive;
73
74/*
75 * Identify the physical page mapped at the given kernel virtual
76 * address. Insert this physical page into the given address space at
77 * the given virtual address, replacing the physical page, if any,
78 * that already exists there.
79 */
80static int
81vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
82{
83 vm_map_t map = mapa;
84 vm_page_t kern_pg, user_pg;
85 vm_object_t uobject;
86 vm_map_entry_t entry;
87 vm_pindex_t upindex;
88 vm_prot_t prot;
89 boolean_t wired;
90
91 KASSERT((uaddr & PAGE_MASK) == 0,
92 ("vm_pgmoveco: uaddr is not page aligned"));
93
94 /*
95 * Herein the physical page is validated and dirtied. It is
96 * unwired in sf_buf_mext().
97 */
98 kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
99 kern_pg->valid = VM_PAGE_BITS_ALL;
100 KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1,
101 ("vm_pgmoveco: kern_pg is not correctly wired"));
102
103 if ((vm_map_lookup(&map, uaddr,
104 VM_PROT_WRITE, &entry, &uobject,
105 &upindex, &prot, &wired)) != KERN_SUCCESS) {
106 return(EFAULT);
107 }
108 VM_OBJECT_WLOCK(uobject);
109retry:
110 if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
111 if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco"))
112 goto retry;
113 vm_page_lock(user_pg);
114 pmap_remove_all(user_pg);
115 vm_page_free(user_pg);
116 vm_page_unlock(user_pg);
117 } else {
118 /*
119 * Even if a physical page does not exist in the
120 * object chain's first object, a physical page from a
121 * backing object may be mapped read only.
122 */
123 if (uobject->backing_object != NULL)
124 pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE);
125 }
126 if (vm_page_insert(kern_pg, uobject, upindex)) {
127 VM_OBJECT_WUNLOCK(uobject);
128 VM_WAIT;
129 VM_OBJECT_WLOCK(uobject);
130 goto retry;
131 }
132 vm_page_dirty(kern_pg);
133 VM_OBJECT_WUNLOCK(uobject);
134 vm_map_lookup_done(map, entry);
135 return(KERN_SUCCESS);
136}
137#endif /* SOCKET_SEND_COW */
138
139int
140copyin_nofault(const void *udaddr, void *kaddr, size_t len)
141{
142 int error, save;
143
144 save = vm_fault_disable_pagefaults();
145 error = copyin(udaddr, kaddr, len);
146 vm_fault_enable_pagefaults(save);

--- 161 unchanged lines hidden (view full) ---

308 return (EINVAL);
309 if (buflen <= 0 || offset >= buflen)
310 return (0);
311 if ((n = buflen - offset) > IOSIZE_MAX)
312 return (EINVAL);
313 return (uiomove((char *)buf + offset, n, uio));
314}
315
65int
66copyin_nofault(const void *udaddr, void *kaddr, size_t len)
67{
68 int error, save;
69
70 save = vm_fault_disable_pagefaults();
71 error = copyin(udaddr, kaddr, len);
72 vm_fault_enable_pagefaults(save);

--- 161 unchanged lines hidden (view full) ---

234 return (EINVAL);
235 if (buflen <= 0 || offset >= buflen)
236 return (0);
237 if ((n = buflen - offset) > IOSIZE_MAX)
238 return (EINVAL);
239 return (uiomove((char *)buf + offset, n, uio));
240}
241
316#ifdef SOCKET_RECV_PFLIP
317/*
242/*
318 * Experimental support for zero-copy I/O
319 */
320static int
321userspaceco(void *cp, u_int cnt, struct uio *uio, int disposable)
322{
323 struct iovec *iov;
324 int error;
325
326 iov = uio->uio_iov;
327 if (uio->uio_rw == UIO_READ) {
328 if ((so_zero_copy_receive != 0)
329 && ((cnt & PAGE_MASK) == 0)
330 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
331 && ((uio->uio_offset & PAGE_MASK) == 0)
332 && ((((intptr_t) cp) & PAGE_MASK) == 0)
333 && (disposable != 0)) {
334 /* SOCKET: use page-trading */
335 /*
336 * We only want to call vm_pgmoveco() on
337 * disposeable pages, since it gives the
338 * kernel page to the userland process.
339 */
340 error = vm_pgmoveco(&curproc->p_vmspace->vm_map,
341 (vm_offset_t)cp, (vm_offset_t)iov->iov_base);
342
343 /*
344 * If we get an error back, attempt
345 * to use copyout() instead. The
346 * disposable page should be freed
347 * automatically if we weren't able to move
348 * it into userland.
349 */
350 if (error != 0)
351 error = copyout(cp, iov->iov_base, cnt);
352 } else {
353 error = copyout(cp, iov->iov_base, cnt);
354 }
355 } else {
356 error = copyin(iov->iov_base, cp, cnt);
357 }
358 return (error);
359}
360
361int
362uiomoveco(void *cp, int n, struct uio *uio, int disposable)
363{
364 struct iovec *iov;
365 u_int cnt;
366 int error;
367
368 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
369 ("uiomoveco: mode"));
370 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
371 ("uiomoveco proc"));
372
373 while (n > 0 && uio->uio_resid) {
374 iov = uio->uio_iov;
375 cnt = iov->iov_len;
376 if (cnt == 0) {
377 uio->uio_iov++;
378 uio->uio_iovcnt--;
379 continue;
380 }
381 if (cnt > n)
382 cnt = n;
383
384 switch (uio->uio_segflg) {
385
386 case UIO_USERSPACE:
387 maybe_yield();
388 error = userspaceco(cp, cnt, uio, disposable);
389 if (error)
390 return (error);
391 break;
392
393 case UIO_SYSSPACE:
394 if (uio->uio_rw == UIO_READ)
395 bcopy(cp, iov->iov_base, cnt);
396 else
397 bcopy(iov->iov_base, cp, cnt);
398 break;
399 case UIO_NOCOPY:
400 break;
401 }
402 iov->iov_base = (char *)iov->iov_base + cnt;
403 iov->iov_len -= cnt;
404 uio->uio_resid -= cnt;
405 uio->uio_offset += cnt;
406 cp = (char *)cp + cnt;
407 n -= cnt;
408 }
409 return (0);
410}
411#endif /* SOCKET_RECV_PFLIP */
412
413/*
414 * Give next character to user as result of read.
415 */
416int
417ureadc(int c, struct uio *uio)
418{
419 struct iovec *iov;
420 char *iov_base;
421

--- 190 unchanged lines hidden --- 		243 * Give next character to user as result of read.
244 */
245int
246ureadc(int c, struct uio *uio)
247{
248 struct iovec *iov;
249 char *iov_base;
250

--- 190 unchanged lines hidden ---