1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27/*	  All Rights Reserved	*/
28
29/*
30 * University Copyright- Copyright (c) 1982, 1986, 1988
31 * The Regents of the University of California
32 * All Rights Reserved
33 *
34 * University Acknowledgment- Portions of this document are derived from
35 * software developed by the University of California, Berkeley, and its
36 * contributors.
37 */
38/*
39 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
40 */
41
42#ifdef _KERNEL
43
44#include <sys/types.h>
45#include <sys/uio_impl.h>
46#include <sys/sysmacros.h>
47#include <sys/strings.h>
48#include <linux/kmap_compat.h>
49#include <linux/uaccess.h>
50
51/*
52 * Move "n" bytes at byte address "p"; "rw" indicates the direction
53 * of the move, and the I/O parameters are provided in "uio", which is
54 * update to reflect the data which was moved.  Returns 0 on success or
55 * a non-zero errno on failure.
56 */
57static int
58zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
59{
60	const struct iovec *iov = uio->uio_iov;
61	size_t skip = uio->uio_skip;
62	ulong_t cnt;
63
64	while (n && uio->uio_resid) {
65		cnt = MIN(iov->iov_len - skip, n);
66		switch (uio->uio_segflg) {
67		case UIO_USERSPACE:
68			/*
69			 * p = kernel data pointer
70			 * iov->iov_base = user data pointer
71			 */
72			if (rw == UIO_READ) {
73				if (copy_to_user(iov->iov_base+skip, p, cnt))
74					return (EFAULT);
75			} else {
76				unsigned long b_left = 0;
77				if (uio->uio_fault_disable) {
78					if (!zfs_access_ok(VERIFY_READ,
79					    (iov->iov_base + skip), cnt)) {
80						return (EFAULT);
81					}
82					pagefault_disable();
83					b_left =
84					    __copy_from_user_inatomic(p,
85					    (iov->iov_base + skip), cnt);
86					pagefault_enable();
87				} else {
88					b_left =
89					    copy_from_user(p,
90					    (iov->iov_base + skip), cnt);
91				}
92				if (b_left > 0) {
93					unsigned long c_bytes =
94					    cnt - b_left;
95					uio->uio_skip += c_bytes;
96					ASSERT3U(uio->uio_skip, <,
97					    iov->iov_len);
98					uio->uio_resid -= c_bytes;
99					uio->uio_loffset += c_bytes;
100					return (EFAULT);
101				}
102			}
103			break;
104		case UIO_SYSSPACE:
105			if (rw == UIO_READ)
106				bcopy(p, iov->iov_base + skip, cnt);
107			else
108				bcopy(iov->iov_base + skip, p, cnt);
109			break;
110		default:
111			ASSERT(0);
112		}
113		skip += cnt;
114		if (skip == iov->iov_len) {
115			skip = 0;
116			uio->uio_iov = (++iov);
117			uio->uio_iovcnt--;
118		}
119		uio->uio_skip = skip;
120		uio->uio_resid -= cnt;
121		uio->uio_loffset += cnt;
122		p = (caddr_t)p + cnt;
123		n -= cnt;
124	}
125	return (0);
126}
127
128static int
129zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
130{
131	const struct bio_vec *bv = uio->uio_bvec;
132	size_t skip = uio->uio_skip;
133	ulong_t cnt;
134
135	while (n && uio->uio_resid) {
136		void *paddr;
137		cnt = MIN(bv->bv_len - skip, n);
138
139		paddr = zfs_kmap_atomic(bv->bv_page);
140		if (rw == UIO_READ)
141			bcopy(p, paddr + bv->bv_offset + skip, cnt);
142		else
143			bcopy(paddr + bv->bv_offset + skip, p, cnt);
144		zfs_kunmap_atomic(paddr);
145
146		skip += cnt;
147		if (skip == bv->bv_len) {
148			skip = 0;
149			uio->uio_bvec = (++bv);
150			uio->uio_iovcnt--;
151		}
152		uio->uio_skip = skip;
153		uio->uio_resid -= cnt;
154		uio->uio_loffset += cnt;
155		p = (caddr_t)p + cnt;
156		n -= cnt;
157	}
158	return (0);
159}
160
161#if defined(HAVE_VFS_IOV_ITER)
162static int
163zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio,
164    boolean_t revert)
165{
166	size_t cnt = MIN(n, uio->uio_resid);
167
168	if (uio->uio_skip)
169		iov_iter_advance(uio->uio_iter, uio->uio_skip);
170
171	if (rw == UIO_READ)
172		cnt = copy_to_iter(p, cnt, uio->uio_iter);
173	else
174		cnt = copy_from_iter(p, cnt, uio->uio_iter);
175
176	/*
177	 * When operating on a full pipe no bytes are processed.
178	 * In which case return EFAULT which is converted to EAGAIN
179	 * by the kernel's generic_file_splice_read() function.
180	 */
181	if (cnt == 0)
182		return (EFAULT);
183
184	/*
185	 * Revert advancing the uio_iter.  This is set by zfs_uiocopy()
186	 * to avoid consuming the uio and its iov_iter structure.
187	 */
188	if (revert)
189		iov_iter_revert(uio->uio_iter, cnt);
190
191	uio->uio_resid -= cnt;
192	uio->uio_loffset += cnt;
193
194	return (0);
195}
196#endif
197
198int
199zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
200{
201	if (uio->uio_segflg == UIO_BVEC)
202		return (zfs_uiomove_bvec(p, n, rw, uio));
203#if defined(HAVE_VFS_IOV_ITER)
204	else if (uio->uio_segflg == UIO_ITER)
205		return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE));
206#endif
207	else
208		return (zfs_uiomove_iov(p, n, rw, uio));
209}
210EXPORT_SYMBOL(zfs_uiomove);
211
212/*
213 * Fault in the pages of the first n bytes specified by the uio structure.
214 * 1 byte in each page is touched and the uio struct is unmodified. Any
215 * error will terminate the process as this is only a best attempt to get
216 * the pages resident.
217 */
218int
219zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
220{
221	if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) {
222		/* There's never a need to fault in kernel pages */
223		return (0);
224#if defined(HAVE_VFS_IOV_ITER)
225	} else if (uio->uio_segflg == UIO_ITER) {
226		/*
227		 * At least a Linux 4.9 kernel, iov_iter_fault_in_readable()
228		 * can be relied on to fault in user pages when referenced.
229		 */
230		if (iov_iter_fault_in_readable(uio->uio_iter, n))
231			return (EFAULT);
232#endif
233	} else {
234		/* Fault in all user pages */
235		ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE);
236		const struct iovec *iov = uio->uio_iov;
237		int iovcnt = uio->uio_iovcnt;
238		size_t skip = uio->uio_skip;
239		uint8_t tmp;
240		caddr_t p;
241
242		for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
243			ulong_t cnt = MIN(iov->iov_len - skip, n);
244			/* empty iov */
245			if (cnt == 0)
246				continue;
247			n -= cnt;
248			/* touch each page in this segment. */
249			p = iov->iov_base + skip;
250			while (cnt) {
251				if (get_user(tmp, (uint8_t *)p))
252					return (EFAULT);
253				ulong_t incr = MIN(cnt, PAGESIZE);
254				p += incr;
255				cnt -= incr;
256			}
257			/* touch the last byte in case it straddles a page. */
258			p--;
259			if (get_user(tmp, (uint8_t *)p))
260				return (EFAULT);
261		}
262	}
263
264	return (0);
265}
266EXPORT_SYMBOL(zfs_uio_prefaultpages);
267
268/*
269 * The same as zfs_uiomove() but doesn't modify uio structure.
270 * return in cbytes how many bytes were copied.
271 */
272int
273zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
274{
275	zfs_uio_t uio_copy;
276	int ret;
277
278	bcopy(uio, &uio_copy, sizeof (zfs_uio_t));
279
280	if (uio->uio_segflg == UIO_BVEC)
281		ret = zfs_uiomove_bvec(p, n, rw, &uio_copy);
282#if defined(HAVE_VFS_IOV_ITER)
283	else if (uio->uio_segflg == UIO_ITER)
284		ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
285#endif
286	else
287		ret = zfs_uiomove_iov(p, n, rw, &uio_copy);
288
289	*cbytes = uio->uio_resid - uio_copy.uio_resid;
290
291	return (ret);
292}
293EXPORT_SYMBOL(zfs_uiocopy);
294
295/*
296 * Drop the next n chars out of *uio.
297 */
298void
299zfs_uioskip(zfs_uio_t *uio, size_t n)
300{
301	if (n > uio->uio_resid)
302		return;
303
304	if (uio->uio_segflg == UIO_BVEC) {
305		uio->uio_skip += n;
306		while (uio->uio_iovcnt &&
307		    uio->uio_skip >= uio->uio_bvec->bv_len) {
308			uio->uio_skip -= uio->uio_bvec->bv_len;
309			uio->uio_bvec++;
310			uio->uio_iovcnt--;
311		}
312#if defined(HAVE_VFS_IOV_ITER)
313	} else if (uio->uio_segflg == UIO_ITER) {
314		iov_iter_advance(uio->uio_iter, n);
315#endif
316	} else {
317		uio->uio_skip += n;
318		while (uio->uio_iovcnt &&
319		    uio->uio_skip >= uio->uio_iov->iov_len) {
320			uio->uio_skip -= uio->uio_iov->iov_len;
321			uio->uio_iov++;
322			uio->uio_iovcnt--;
323		}
324	}
325	uio->uio_loffset += n;
326	uio->uio_resid -= n;
327}
328EXPORT_SYMBOL(zfs_uioskip);
329
330#endif /* _KERNEL */
331