1/*	$NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $	*/
2
3/*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
9 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
10 *
11 * This material is based upon work supported by the Defense Advanced Research
12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
13 * Contract No. N66001-09-C-2073.
14 * Approved for Public Release, Distribution Unlimited
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 *    notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 *    notice, this list of conditions and the following disclaimer in the
23 *    documentation and/or other materials provided with the distribution.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38#define	__UFETCHSTORE_PRIVATE
39
40#include <sys/cdefs.h>
41__KERNEL_RCSID(0, "$NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $");
42
43#include <sys/param.h>
44#include <sys/lwp.h>
45#include <sys/systm.h>
46
47#include <powerpc/pcb.h>
48
49#include <powerpc/booke/cpuvar.h>
50
51static inline uint8_t
52copyin_byte(const uint8_t * const usaddr8, register_t ds_msr)
53{
54	register_t msr;
55	uint8_t data;
56	__asm volatile(
57		"mfmsr	%[msr]; "			/* Save MSR */
58		"mtmsr	%[ds_msr]; sync; isync; "	/* DS on */
59		"lbz	%[data],0(%[usaddr8]); "	/* fetch user byte */
60		"mtmsr	%[msr]; sync; isync; "		/* DS off */
61	    : [msr] "=&r" (msr), [data] "=r" (data)
62	    : [ds_msr] "r" (ds_msr), [usaddr8] "b" (usaddr8));
63	return data;
64}
65
66static inline uint16_t
67copyin_halfword(const uint16_t * const usaddr16, register_t ds_msr)
68{
69	register_t msr;
70	uint16_t data;
71	__asm volatile(
72		"mfmsr	%[msr]; "			/* Save MSR */
73		"mtmsr	%[ds_msr]; sync; isync; "	/* DS on */
74		"lhz	%[data],0(%[usaddr16]); "	/* fetch user byte */
75		"mtmsr	%[msr]; sync; isync; "		/* DS off */
76	    : [msr] "=&r" (msr), [data] "=r" (data)
77	    : [ds_msr] "r" (ds_msr), [usaddr16] "b" (usaddr16));
78	return data;
79}
80
81static inline uint32_t
82copyin_word(const uint32_t * const usaddr32, register_t ds_msr)
83{
84	register_t msr;
85	uint32_t data;
86	__asm volatile(
87		"mfmsr	%[msr]; "			/* Save MSR */
88		"mtmsr	%[ds_msr]; sync; isync; "	/* DS on */
89		"lwz	%[data],0(%[usaddr32]); "	/* load user byte */
90		"mtmsr	%[msr]; sync; isync; "		/* DS off */
91	    : [msr] "=&r" (msr), [data] "=r" (data)
92	    : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
93	return data;
94}
95
96static inline uint32_t
97copyin_word_bswap(const uint32_t * const usaddr32, register_t ds_msr)
98{
99	register_t msr;
100	uint32_t data;
101	__asm volatile(
102		"mfmsr	%[msr]; "			/* Save MSR */
103		"mtmsr	%[ds_msr]; sync; isync; "	/* DS on */
104		"lwbrx	%[data],0,%[usaddr32]; "	/* load user LE word */
105		"mtmsr	%[msr]; sync; isync; "		/* DS off */
106	    : [msr] "=&r" (msr), [data] "=r" (data)
107	    : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
108	return data;
109}
110
111static inline void
112copyin_8words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr)
113{
114	register_t msr;
115	//uint32_t data[8];
116	__asm volatile(
117		"mfmsr	%[msr]"				/* Save MSR */
118	"\n\t"	"mtmsr	%[ds_msr]; sync; isync"		/* DS on */
119	"\n\t"	"lwz	%[data0],0(%[usaddr32])"	/* fetch user data */
120	"\n\t"	"lwz	%[data1],4(%[usaddr32])"	/* fetch user data */
121	"\n\t"	"lwz	%[data2],8(%[usaddr32])"	/* fetch user data */
122	"\n\t"	"lwz	%[data3],12(%[usaddr32])"	/* fetch user data */
123	"\n\t"	"lwz	%[data4],16(%[usaddr32])"	/* fetch user data */
124	"\n\t"	"lwz	%[data5],20(%[usaddr32])"	/* fetch user data */
125	"\n\t"	"lwz	%[data6],24(%[usaddr32])"	/* fetch user data */
126	"\n\t"	"lwz	%[data7],28(%[usaddr32])"	/* fetch user data */
127	"\n\t"	"mtmsr	%[msr]; sync; isync"		/* DS off */
128	    : [msr] "=&r" (msr),
129	      [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]),
130	      [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]),
131	      [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]),
132	      [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7])
133	    : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
134}
135
136static inline void
137copyin_16words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr)
138{
139	register_t msr;
140	__asm volatile(
141		"mfmsr	%[msr]"				/* Save MSR */
142	"\n\t"	"mtmsr	%[ds_msr]; sync; isync"		/* DS on */
143	"\n\t"	"lwz	%[data0],0(%[usaddr32])"	/* fetch user data */
144	"\n\t"	"lwz	%[data1],4(%[usaddr32])"	/* fetch user data */
145	"\n\t"	"lwz	%[data2],8(%[usaddr32])"	/* fetch user data */
146	"\n\t"	"lwz	%[data3],12(%[usaddr32])"	/* fetch user data */
147	"\n\t"	"lwz	%[data4],16(%[usaddr32])"	/* fetch user data */
148	"\n\t"	"lwz	%[data5],20(%[usaddr32])"	/* fetch user data */
149	"\n\t"	"lwz	%[data6],24(%[usaddr32])"	/* fetch user data */
150	"\n\t"	"lwz	%[data7],28(%[usaddr32])"	/* fetch user data */
151	"\n\t"	"lwz	%[data8],32(%[usaddr32])"	/* fetch user data */
152	"\n\t"	"lwz	%[data9],36(%[usaddr32])"	/* fetch user data */
153	"\n\t"	"lwz	%[data10],40(%[usaddr32])"	/* fetch user data */
154	"\n\t"	"lwz	%[data11],44(%[usaddr32])"	/* fetch user data */
155	"\n\t"	"lwz	%[data12],48(%[usaddr32])"	/* fetch user data */
156	"\n\t"	"lwz	%[data13],52(%[usaddr32])"	/* fetch user data */
157	"\n\t"	"lwz	%[data14],56(%[usaddr32])"	/* fetch user data */
158	"\n\t"	"lwz	%[data15],60(%[usaddr32])"	/* fetch user data */
159	"\n\t"	"mtmsr	%[msr]; sync; isync"		/* DS off */
160	    : [msr] "=&r" (msr),
161	      [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]),
162	      [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]),
163	      [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]),
164	      [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]),
165	      [data8] "=&r" (kdaddr32[8]), [data9] "=&r" (kdaddr32[9]),
166	      [data10] "=&r" (kdaddr32[10]), [data11] "=&r" (kdaddr32[11]),
167	      [data12] "=&r" (kdaddr32[12]), [data13] "=&r" (kdaddr32[13]),
168	      [data14] "=&r" (kdaddr32[14]), [data15] "=&r" (kdaddr32[15])
169	    : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
170}
171static inline void
172copyin_bytes(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr)
173{
174	const uint8_t *usaddr8 = (void *)usaddr;
175	uint8_t *kdaddr8 = (void *)kdaddr;
176	while (len-- > 0) {
177		*kdaddr8++ = copyin_byte(usaddr8++, ds_msr);
178	}
179}
180
181static inline void
182copyin_words(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr)
183{
184	KASSERT((kdaddr & 3) == 0);
185	KASSERT((usaddr & 3) == 0);
186	const uint32_t *usaddr32 = (void *)usaddr;
187	uint32_t *kdaddr32 = (void *)kdaddr;
188	len >>= 2;
189	while (len >= 16) {
190		copyin_16words(usaddr32, kdaddr32, ds_msr);
191		usaddr32 += 16, kdaddr32 += 16, len -= 16;
192	}
193	KASSERT(len < 16);
194	if (len >= 8) {
195		copyin_8words(usaddr32, kdaddr32, ds_msr);
196		usaddr32 += 8, kdaddr32 += 8, len -= 8;
197	}
198	while (len-- > 0) {
199		*kdaddr32++ = copyin_word(usaddr32++, ds_msr);
200	}
201}
202
203int
204_ufetch_8(const uint8_t *vusaddr, uint8_t *valp)
205{
206	struct pcb * const pcb = lwp_getpcb(curlwp);
207	struct faultbuf env;
208
209	if (setfault(&env) != 0) {
210		pcb->pcb_onfault = NULL;
211		return EFAULT;
212	}
213
214	*valp = copyin_byte(vusaddr, mfmsr() | PSL_DS);
215
216	pcb->pcb_onfault = NULL;
217
218	return 0;
219}
220
221int
222_ufetch_16(const uint16_t *vusaddr, uint16_t *valp)
223{
224	struct pcb * const pcb = lwp_getpcb(curlwp);
225	struct faultbuf env;
226
227	if (setfault(&env) != 0) {
228		pcb->pcb_onfault = NULL;
229		return EFAULT;
230	}
231
232	*valp = copyin_halfword(vusaddr, mfmsr() | PSL_DS);
233
234	pcb->pcb_onfault = NULL;
235
236	return 0;
237}
238
239int
240_ufetch_32(const uint32_t *vusaddr, uint32_t *valp)
241{
242	struct pcb * const pcb = lwp_getpcb(curlwp);
243	struct faultbuf env;
244
245	if (setfault(&env) != 0) {
246		pcb->pcb_onfault = NULL;
247		return EFAULT;
248	}
249
250	*valp = copyin_word(vusaddr, mfmsr() | PSL_DS);
251
252	pcb->pcb_onfault = NULL;
253
254	return 0;
255}
256
257int
258copyin(const void *vusaddr, void *vkdaddr, size_t len)
259{
260	struct pcb * const pcb = lwp_getpcb(curlwp);
261	struct faultbuf env;
262	vaddr_t usaddr = (vaddr_t) vusaddr;
263	vaddr_t kdaddr = (vaddr_t) vkdaddr;
264
265	if (__predict_false(len == 0)) {
266		return 0;
267	}
268
269	const register_t ds_msr = mfmsr() | PSL_DS;
270
271	int rv = setfault(&env);
272	if (rv != 0) {
273		pcb->pcb_onfault = NULL;
274		return rv;
275	}
276
277	if (__predict_false(len < 4)) {
278		copyin_bytes(usaddr, kdaddr, len, ds_msr);
279		pcb->pcb_onfault = NULL;
280		return 0;
281	}
282
283	const size_t alignment = (usaddr ^ kdaddr) & 3;
284	if (__predict_true(alignment == 0)) {
285		size_t slen;
286		if (__predict_false(kdaddr & 3)) {
287			slen = 4 - (kdaddr & 3);
288			copyin_bytes(usaddr, kdaddr, slen, ds_msr);
289			usaddr += slen, kdaddr += slen, len -= slen;
290		}
291		slen = len & ~3;
292		if (__predict_true(slen >= 4)) {
293			copyin_words(usaddr, kdaddr, slen, ds_msr);
294			usaddr += slen, kdaddr += slen, len -= slen;
295		}
296	}
297	if (len > 0) {
298		copyin_bytes(usaddr, kdaddr, len, ds_msr);
299	}
300	pcb->pcb_onfault = NULL;
301	return 0;
302}
303
304int
305copyinstr(const void *usaddr, void *kdaddr, size_t len, size_t *done)
306{
307	struct pcb * const pcb = lwp_getpcb(curlwp);
308	struct faultbuf env;
309	int rv;
310
311	if (__predict_false(len == 0)) {
312		if (done)
313			*done = 0;
314		return 0;
315	}
316
317	rv = setfault(&env);
318	if (rv != 0) {
319		pcb->pcb_onfault = NULL;
320		if (done)
321			*done = 0;
322		return rv;
323	}
324
325	const register_t ds_msr = mfmsr() | PSL_DS;
326	const uint32_t *usaddr32 = (const void *)((uintptr_t)usaddr & ~3);
327	uint8_t *kdaddr8 = kdaddr;
328	size_t copylen, wlen;
329	uint32_t data;
330	size_t uoff = (uintptr_t)usaddr & 3;
331	wlen = 4 - uoff;
332	/*
333	 * We need discard any leading bytes if the address was
334	 * unaligned.  We read the words byteswapped so that the LSB
335	 * contains the lowest address byte.
336	 */
337	data = copyin_word_bswap(usaddr32++, ds_msr) >> (8 * uoff);
338	for (copylen = 0; copylen < len; copylen++, wlen--, data >>= 8) {
339		if (wlen == 0) {
340			/*
341			 * If we've depleted the data in the word, fetch the
342			 * next one.
343			 */
344			data = copyin_word_bswap(usaddr32++, ds_msr);
345			wlen = 4;
346		}
347		*kdaddr8++ = data;
348		if ((uint8_t) data == 0) {
349			copylen++;
350			goto out;
351		}
352	}
353	rv = ENAMETOOLONG;
354
355out:
356	pcb->pcb_onfault = NULL;
357	if (done)
358		*done = copylen;
359	return rv;
360}
361