1/*	$NetBSD: bcopyinout_xscale.S,v 1.3 2003/12/15 09:27:18 scw Exp $	*/
2
3/*-
4 * Copyright 2003 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Steve C. Woodford for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 *    must display the following acknowledgement:
19 *      This product includes software developed for the NetBSD Project by
20 *      Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 *    or promote products derived from this software without specific prior
23 *    written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38#include <arm/asm_help.h>
39#include <assym.s>
40#include <mach/arm/asm.h>
41
42#define ENAMETOOLONG    0x3f
43#define SAVE_REGS       stmfd	sp!, {r4-r6}
44#define RESTORE_REGS	ldmfd	sp!, {r4-r6}
45
46.code 32
47.arm
48
49/*
50 * r0 = user space address
51 * r1 = kernel space address
52 * r2 = length
53 *
54 * Copies bytes from user space to kernel space
55 */
56EnterARM(copyin)
57EnterARM(copyinmsg)
58    cmp     r2, #0
59    moveq   r0, #0
60    bxeq    lr
61    cmp     r0, #0x80000000
62    bcs     copyio_kernel
63    stmfd   sp!,{r10,r11,lr}
64    mov     r3, #0
65    LoadThreadRegister(r10)
66    adr     r12, .Lcopyin_fault
67    str     r12, [r10, TH_RECOVER]
68    bl      .Lcopyin_guts
69    mov     r11, #0
70    str     r11, [r10, TH_RECOVER]
71    mov     r0, #0
72    ldmfd   sp!,{r10,r11,pc}
73
74.Lcopyin_fault:
75    mov     r11, #0
76    str     r11, [r10, TH_RECOVER]
77	mov	r0, #0xE    /* EFAULT */
78	cmp	r3, #0x00
79	ldmgtfd	sp!, {r4-r7}		/* r3 > 0 Restore r4-r7 */
80	ldmltfd	sp!, {r4-r9}		/* r3 < 0 Restore r4-r9 */
81	ldmfd	sp!, {r10-r11, pc}
82
83.Lcopyin_guts:
84	pld	[r0]
85	/* Word-align the destination buffer */
86	ands	ip, r1, #0x03		/* Already word aligned? */
87	beq	.Lcopyin_wordaligned	/* Yup */
88	rsb	ip, ip, #0x04
89	cmp	r2, ip			/* Enough bytes left to align it? */
90	blt	.Lcopyin_l4_2		/* Nope. Just copy bytewise */
91	sub	r2, r2, ip
92	rsbs	ip, ip, #0x03
93	addne	pc, pc, ip, lsl #3
94	nop
95	ldrbt	ip, [r0], #0x01
96	strb	ip, [r1], #0x01
97	ldrbt	ip, [r0], #0x01
98	strb	ip, [r1], #0x01
99	ldrbt	ip, [r0], #0x01
100	strb	ip, [r1], #0x01
101	cmp	r2, #0x00		/* All done? */
102	RETeq
103
104	/* Destination buffer is now word aligned */
105.Lcopyin_wordaligned:
106	ands	ip, r0, #0x03		/* Is src also word-aligned? */
107	bne	.Lcopyin_bad_align	/* Nope. Things just got bad */
108	cmp	r2, #0x08		/* Less than 8 bytes remaining? */
109	blt	.Lcopyin_w_less_than8
110
111	/* Quad-align the destination buffer */
112	tst	r1, #0x07		/* Already quad aligned? */
113	ldrnet	ip, [r0], #0x04
114	strne	ip, [r1], #0x04
115	subne	r2, r2, #0x04
116	stmfd	sp!, {r4-r9}		/* Free up some registers */
117	mov	r3, #-1			/* Signal restore r4-r9 */
118
119	/* Destination buffer quad aligned, source is word aligned */
120	subs	r2, r2, #0x80
121	blt	.Lcopyin_w_lessthan128
122
123	/* Copy 128 bytes at a time */
124.Lcopyin_w_loop128:
125	ldrt	r4, [r0], #0x04		/* LD:00-03 */
126	ldrt	r5, [r0], #0x04		/* LD:04-07 */
127	pld	[r0, #0x18]		/* Prefetch 0x20 */
128	ldrt	r6, [r0], #0x04		/* LD:08-0b */
129	ldrt	r7, [r0], #0x04		/* LD:0c-0f */
130	ldrt	r8, [r0], #0x04		/* LD:10-13 */
131	ldrt	r9, [r0], #0x04		/* LD:14-17 */
132	strd	r4, [r1], #0x08		/* ST:00-07 */
133	ldrt	r4, [r0], #0x04		/* LD:18-1b */
134	ldrt	r5, [r0], #0x04		/* LD:1c-1f */
135	strd	r6, [r1], #0x08		/* ST:08-0f */
136	ldrt	r6, [r0], #0x04		/* LD:20-23 */
137	ldrt	r7, [r0], #0x04		/* LD:24-27 */
138	pld	[r0, #0x18]		/* Prefetch 0x40 */
139	strd	r8, [r1], #0x08		/* ST:10-17 */
140	ldrt	r8, [r0], #0x04		/* LD:28-2b */
141	ldrt	r9, [r0], #0x04		/* LD:2c-2f */
142	strd	r4, [r1], #0x08		/* ST:18-1f */
143	ldrt	r4, [r0], #0x04		/* LD:30-33 */
144	ldrt	r5, [r0], #0x04		/* LD:34-37 */
145	strd	r6, [r1], #0x08		/* ST:20-27 */
146	ldrt	r6, [r0], #0x04		/* LD:38-3b */
147	ldrt	r7, [r0], #0x04		/* LD:3c-3f */
148	strd	r8, [r1], #0x08		/* ST:28-2f */
149	ldrt	r8, [r0], #0x04		/* LD:40-43 */
150	ldrt	r9, [r0], #0x04		/* LD:44-47 */
151	pld	[r0, #0x18]		/* Prefetch 0x60 */
152	strd	r4, [r1], #0x08		/* ST:30-37 */
153	ldrt	r4, [r0], #0x04		/* LD:48-4b */
154	ldrt	r5, [r0], #0x04		/* LD:4c-4f */
155	strd	r6, [r1], #0x08		/* ST:38-3f */
156	ldrt	r6, [r0], #0x04		/* LD:50-53 */
157	ldrt	r7, [r0], #0x04		/* LD:54-57 */
158	strd	r8, [r1], #0x08		/* ST:40-47 */
159	ldrt	r8, [r0], #0x04		/* LD:58-5b */
160	ldrt	r9, [r0], #0x04		/* LD:5c-5f */
161	strd	r4, [r1], #0x08		/* ST:48-4f */
162	ldrt	r4, [r0], #0x04		/* LD:60-63 */
163	ldrt	r5, [r0], #0x04		/* LD:64-67 */
164	pld	[r0, #0x18]		/* Prefetch 0x80 */
165	strd	r6, [r1], #0x08		/* ST:50-57 */
166	ldrt	r6, [r0], #0x04		/* LD:68-6b */
167	ldrt	r7, [r0], #0x04		/* LD:6c-6f */
168	strd	r8, [r1], #0x08		/* ST:58-5f */
169	ldrt	r8, [r0], #0x04		/* LD:70-73 */
170	ldrt	r9, [r0], #0x04		/* LD:74-77 */
171	strd	r4, [r1], #0x08		/* ST:60-67 */
172	ldrt	r4, [r0], #0x04		/* LD:78-7b */
173	ldrt	r5, [r0], #0x04		/* LD:7c-7f */
174	strd	r6, [r1], #0x08		/* ST:68-6f */
175	strd	r8, [r1], #0x08		/* ST:70-77 */
176	subs	r2, r2, #0x80
177	strd	r4, [r1], #0x08		/* ST:78-7f */
178	bge	.Lcopyin_w_loop128
179
180.Lcopyin_w_lessthan128:
181	adds	r2, r2, #0x80		/* Adjust for extra sub */
182	ldmeqfd	sp!, {r4-r9}
183	RETeq
184	subs	r2, r2, #0x20
185	blt	.Lcopyin_w_lessthan32
186
187	/* Copy 32 bytes at a time */
188.Lcopyin_w_loop32:
189	ldrt	r4, [r0], #0x04
190	ldrt	r5, [r0], #0x04
191	pld	[r0, #0x18]
192	ldrt	r6, [r0], #0x04
193	ldrt	r7, [r0], #0x04
194	ldrt	r8, [r0], #0x04
195	ldrt	r9, [r0], #0x04
196	strd	r4, [r1], #0x08
197	ldrt	r4, [r0], #0x04
198	ldrt	r5, [r0], #0x04
199	strd	r6, [r1], #0x08
200	strd	r8, [r1], #0x08
201	subs	r2, r2, #0x20
202	strd	r4, [r1], #0x08
203	bge	.Lcopyin_w_loop32
204
205.Lcopyin_w_lessthan32:
206	adds	r2, r2, #0x20		/* Adjust for extra sub */
207	ldmeqfd	sp!, {r4-r9}
208	RETeq				/* Return now if done */
209
210	and	r4, r2, #0x18
211	rsb	r5, r4, #0x18
212	subs	r2, r2, r4
213	add	pc, pc, r5, lsl #1
214	nop
215
216	/* At least 24 bytes remaining */
217	ldrt	r4, [r0], #0x04
218	ldrt	r5, [r0], #0x04
219	nop
220	strd	r4, [r1], #0x08
221
222	/* At least 16 bytes remaining */
223	ldrt	r4, [r0], #0x04
224	ldrt	r5, [r0], #0x04
225	nop
226	strd	r4, [r1], #0x08
227
228	/* At least 8 bytes remaining */
229	ldrt	r4, [r0], #0x04
230	ldrt	r5, [r0], #0x04
231	nop
232	strd	r4, [r1], #0x08
233
234	/* Less than 8 bytes remaining */
235	ldmfd	sp!, {r4-r9}
236	RETeq				/* Return now if done */
237	mov	r3, #0x00
238
239.Lcopyin_w_less_than8:
240	subs	r2, r2, #0x04
241	ldrget	ip, [r0], #0x04
242	strge	ip, [r1], #0x04
243	RETeq				/* Return now if done */
244	addlt	r2, r2, #0x04
245	ldrbt	ip, [r0], #0x01
246	cmp	r2, #0x02
247	ldrgebt	r2, [r0], #0x01
248	strb	ip, [r1], #0x01
249	ldrgtbt	ip, [r0]
250	strgeb	r2, [r1], #0x01
251	strgtb	ip, [r1]
252	RET
253
254/*
255 * At this point, it has not been possible to word align both buffers.
256 * The destination buffer (r1) is word aligned, but the source buffer
257 * (r0) is not.
258 */
259.Lcopyin_bad_align:
260	stmfd	sp!, {r4-r7}
261	mov	r3, #0x01
262	bic	r0, r0, #0x03
263	cmp	ip, #2
264	ldrt	ip, [r0], #0x04
265	bgt	.Lcopyin_bad3
266	beq	.Lcopyin_bad2
267	b	.Lcopyin_bad1
268
269.Lcopyin_bad1_loop16:
270#ifdef __ARMEB__
271	mov	r4, ip, lsl #8
272#else
273	mov	r4, ip, lsr #8
274#endif
275	ldrt	r5, [r0], #0x04
276	pld	[r0, #0x018]
277	ldrt	r6, [r0], #0x04
278	ldrt	r7, [r0], #0x04
279	ldrt	ip, [r0], #0x04
280#ifdef __ARMEB__
281	orr	r4, r4, r5, lsr #24
282	mov	r5, r5, lsl #8
283	orr	r5, r5, r6, lsr #24
284	mov	r6, r6, lsl #8
285	orr	r6, r6, r7, lsr #24
286	mov	r7, r7, lsl #8
287	orr	r7, r7, ip, lsr #24
288#else
289	orr	r4, r4, r5, lsl #24
290	mov	r5, r5, lsr #8
291	orr	r5, r5, r6, lsl #24
292	mov	r6, r6, lsr #8
293	orr	r6, r6, r7, lsl #24
294	mov	r7, r7, lsr #8
295	orr	r7, r7, ip, lsl #24
296#endif
297	str	r4, [r1], #0x04
298	str	r5, [r1], #0x04
299	str	r6, [r1], #0x04
300	str	r7, [r1], #0x04
301.Lcopyin_bad1:
302	subs	r2, r2, #0x10
303	bge	.Lcopyin_bad1_loop16
304
305	adds	r2, r2, #0x10
306	ldmeqfd	sp!, {r4-r7}
307	RETeq				/* Return now if done */
308	subs	r2, r2, #0x04
309	sublt	r0, r0, #0x03
310	blt	.Lcopyin_l4
311
312.Lcopyin_bad1_loop4:
313#ifdef __ARMEB__
314	mov	r4, ip, lsl #8
315#else
316	mov	r4, ip, lsr #8
317#endif
318	ldrt	ip, [r0], #0x04
319	subs	r2, r2, #0x04
320#ifdef __ARMEB__
321	orr	r4, r4, ip, lsr #24
322#else
323	orr	r4, r4, ip, lsl #24
324#endif
325	str	r4, [r1], #0x04
326	bge	.Lcopyin_bad1_loop4
327	sub	r0, r0, #0x03
328	b	.Lcopyin_l4
329
330.Lcopyin_bad2_loop16:
331#ifdef __ARMEB__
332	mov	r4, ip, lsl #16
333#else
334	mov	r4, ip, lsr #16
335#endif
336	ldrt	r5, [r0], #0x04
337	pld	[r0, #0x018]
338	ldrt	r6, [r0], #0x04
339	ldrt	r7, [r0], #0x04
340	ldrt	ip, [r0], #0x04
341#ifdef __ARMEB__
342	orr	r4, r4, r5, lsr #16
343	mov	r5, r5, lsl #16
344	orr	r5, r5, r6, lsr #16
345	mov	r6, r6, lsl #16
346	orr	r6, r6, r7, lsr #16
347	mov	r7, r7, lsl #16
348	orr	r7, r7, ip, lsr #16
349#else
350	orr	r4, r4, r5, lsl #16
351	mov	r5, r5, lsr #16
352	orr	r5, r5, r6, lsl #16
353	mov	r6, r6, lsr #16
354	orr	r6, r6, r7, lsl #16
355	mov	r7, r7, lsr #16
356	orr	r7, r7, ip, lsl #16
357#endif
358	str	r4, [r1], #0x04
359	str	r5, [r1], #0x04
360	str	r6, [r1], #0x04
361	str	r7, [r1], #0x04
362.Lcopyin_bad2:
363	subs	r2, r2, #0x10
364	bge	.Lcopyin_bad2_loop16
365
366	adds	r2, r2, #0x10
367	ldmeqfd	sp!, {r4-r7}
368	RETeq				/* Return now if done */
369	subs	r2, r2, #0x04
370	sublt	r0, r0, #0x02
371	blt	.Lcopyin_l4
372
373.Lcopyin_bad2_loop4:
374#ifdef __ARMEB__
375	mov	r4, ip, lsl #16
376#else
377	mov	r4, ip, lsr #16
378#endif
379	ldrt	ip, [r0], #0x04
380	subs	r2, r2, #0x04
381#ifdef __ARMEB__
382	orr	r4, r4, ip, lsr #16
383#else
384	orr	r4, r4, ip, lsl #16
385#endif
386	str	r4, [r1], #0x04
387	bge	.Lcopyin_bad2_loop4
388	sub	r0, r0, #0x02
389	b	.Lcopyin_l4
390
391.Lcopyin_bad3_loop16:
392#ifdef __ARMEB__
393	mov	r4, ip, lsl #24
394#else
395	mov	r4, ip, lsr #24
396#endif
397	ldrt	r5, [r0], #0x04
398	pld	[r0, #0x018]
399	ldrt	r6, [r0], #0x04
400	ldrt	r7, [r0], #0x04
401	ldrt	ip, [r0], #0x04
402#ifdef __ARMEB__
403	orr	r4, r4, r5, lsr #8
404	mov	r5, r5, lsl #24
405	orr	r5, r5, r6, lsr #8
406	mov	r6, r6, lsl #24
407	orr	r6, r6, r7, lsr #8
408	mov	r7, r7, lsl #24
409	orr	r7, r7, ip, lsr #8
410#else
411	orr	r4, r4, r5, lsl #8
412	mov	r5, r5, lsr #24
413	orr	r5, r5, r6, lsl #8
414	mov	r6, r6, lsr #24
415	orr	r6, r6, r7, lsl #8
416	mov	r7, r7, lsr #24
417	orr	r7, r7, ip, lsl #8
418#endif
419	str	r4, [r1], #0x04
420	str	r5, [r1], #0x04
421	str	r6, [r1], #0x04
422	str	r7, [r1], #0x04
423.Lcopyin_bad3:
424	subs	r2, r2, #0x10
425	bge	.Lcopyin_bad3_loop16
426
427	adds	r2, r2, #0x10
428	ldmeqfd	sp!, {r4-r7}
429	RETeq				/* Return now if done */
430	subs	r2, r2, #0x04
431	sublt	r0, r0, #0x01
432	blt	.Lcopyin_l4
433
434.Lcopyin_bad3_loop4:
435#ifdef __ARMEB__
436	mov	r4, ip, lsl #24
437#else
438	mov	r4, ip, lsr #24
439#endif
440	ldrt	ip, [r0], #0x04
441	subs	r2, r2, #0x04
442#ifdef __ARMEB__
443	orr	r4, r4, ip, lsr #8
444#else
445	orr	r4, r4, ip, lsl #8
446#endif
447	str	r4, [r1], #0x04
448	bge	.Lcopyin_bad3_loop4
449	sub	r0, r0, #0x01
450
451.Lcopyin_l4:
452	ldmfd	sp!, {r4-r7}
453	mov	r3, #0x00
454	adds	r2, r2, #0x04
455	RETeq
456.Lcopyin_l4_2:
457	rsbs	r2, r2, #0x03
458	addne	pc, pc, r2, lsl #3
459	nop
460	ldrbt	ip, [r0], #0x01
461	strb	ip, [r1], #0x01
462	ldrbt	ip, [r0], #0x01
463	strb	ip, [r1], #0x01
464	ldrbt	ip, [r0]
465	strb	ip, [r1]
466	RET
467
468/*
469 * r0 = kernel space address
470 * r1 = user space address
471 * r2 = length
472 *
473 * Copies bytes from kernel space to user space
474 */
475EnterARM(copyout)
476EnterARM(copyoutmsg)
477    cmp r2, #0
478    moveq r0, #0
479    bxeq  lr
480    cmp   r1, #0x80000000
481    bcs copyio_kernel
482    stmfd sp!,{r10,r11,lr}
483    LoadThreadRegister(r10)
484    mov     r3, #0
485    adr     r12, .Lcopyout_fault
486    ldr     r11, [r10, TH_RECOVER]
487    str     r12, [r10, TH_RECOVER]
488    bl  .Lcopyout_guts
489    mov     r11, #0
490    str     r11, [r10, TH_RECOVER]
491    mov r0, #0
492    ldmfd   sp!,{r10,r11,pc}
493
494.Lcopyout_fault:
495    mov     r11, #0
496    str     r11, [r10, TH_RECOVER]
497	mov r0, #0x0e
498	cmp	r3, #0x00
499	ldmgtfd	sp!, {r4-r7}		/* r3 > 0 Restore r4-r7 */
500	ldmltfd	sp!, {r4-r9}		/* r3 < 0 Restore r4-r9 */
501	ldmfd	sp!, {r10-r11, pc}
502
503.Lcopyout_guts:
504	pld	[r0]
505	/* Word-align the destination buffer */
506	ands	ip, r1, #0x03		/* Already word aligned? */
507	beq	.Lcopyout_wordaligned	/* Yup */
508	rsb	ip, ip, #0x04
509	cmp	r2, ip			/* Enough bytes left to align it? */
510	blt	.Lcopyout_l4_2		/* Nope. Just copy bytewise */
511	sub	r2, r2, ip
512	rsbs	ip, ip, #0x03
513	addne	pc, pc, ip, lsl #3
514	nop
515	ldrb	ip, [r0], #0x01
516	strbt	ip, [r1], #0x01
517	ldrb	ip, [r0], #0x01
518	strbt	ip, [r1], #0x01
519	ldrb	ip, [r0], #0x01
520	strbt	ip, [r1], #0x01
521	cmp	r2, #0x00		/* All done? */
522	RETeq
523
524	/* Destination buffer is now word aligned */
525.Lcopyout_wordaligned:
526	ands	ip, r0, #0x03		/* Is src also word-aligned? */
527	bne	.Lcopyout_bad_align	/* Nope. Things just got bad */
528	cmp	r2, #0x08		/* Less than 8 bytes remaining? */
529	blt	.Lcopyout_w_less_than8
530
531	/* Quad-align the destination buffer */
532	tst	r0, #0x07		/* Already quad aligned? */
533	ldrne	ip, [r0], #0x04
534	subne	r2, r2, #0x04
535	strnet	ip, [r1], #0x04
536
537	stmfd	sp!, {r4-r9}		/* Free up some registers */
538	mov	r3, #-1			/* Signal restore r4-r9 */
539
540	/* Destination buffer word aligned, source is quad aligned */
541	subs	r2, r2, #0x80
542	blt	.Lcopyout_w_lessthan128
543
544	/* Copy 128 bytes at a time */
545.Lcopyout_w_loop128:
546	ldrd	r4, [r0], #0x08		/* LD:00-07 */
547	pld	[r0, #0x18]		/* Prefetch 0x20 */
548	ldrd	r6, [r0], #0x08		/* LD:08-0f */
549	ldrd	r8, [r0], #0x08		/* LD:10-17 */
550	strt	r4, [r1], #0x04		/* ST:00-03 */
551	strt	r5, [r1], #0x04		/* ST:04-07 */
552	ldrd	r4, [r0], #0x08		/* LD:18-1f */
553	strt	r6, [r1], #0x04		/* ST:08-0b */
554	strt	r7, [r1], #0x04		/* ST:0c-0f */
555	ldrd	r6, [r0], #0x08		/* LD:20-27 */
556	pld	[r0, #0x18]		/* Prefetch 0x40 */
557	strt	r8, [r1], #0x04		/* ST:10-13 */
558	strt	r9, [r1], #0x04		/* ST:14-17 */
559	ldrd	r8, [r0], #0x08		/* LD:28-2f */
560	strt	r4, [r1], #0x04		/* ST:18-1b */
561	strt	r5, [r1], #0x04		/* ST:1c-1f */
562	ldrd	r4, [r0], #0x08		/* LD:30-37 */
563	strt	r6, [r1], #0x04		/* ST:20-23 */
564	strt	r7, [r1], #0x04		/* ST:24-27 */
565	ldrd	r6, [r0], #0x08		/* LD:38-3f */
566	strt	r8, [r1], #0x04		/* ST:28-2b */
567	strt	r9, [r1], #0x04		/* ST:2c-2f */
568	ldrd	r8, [r0], #0x08		/* LD:40-47 */
569	pld	[r0, #0x18]		/* Prefetch 0x60 */
570	strt	r4, [r1], #0x04		/* ST:30-33 */
571	strt	r5, [r1], #0x04		/* ST:34-37 */
572	ldrd	r4, [r0], #0x08		/* LD:48-4f */
573	strt	r6, [r1], #0x04		/* ST:38-3b */
574	strt	r7, [r1], #0x04		/* ST:3c-3f */
575	ldrd	r6, [r0], #0x08		/* LD:50-57 */
576	strt	r8, [r1], #0x04		/* ST:40-43 */
577	strt	r9, [r1], #0x04		/* ST:44-47 */
578	ldrd	r8, [r0], #0x08		/* LD:58-4f */
579	strt	r4, [r1], #0x04		/* ST:48-4b */
580	strt	r5, [r1], #0x04		/* ST:4c-4f */
581	ldrd	r4, [r0], #0x08		/* LD:60-67 */
582	pld	[r0, #0x18]		/* Prefetch 0x80 */
583	strt	r6, [r1], #0x04		/* ST:50-53 */
584	strt	r7, [r1], #0x04		/* ST:54-57 */
585	ldrd	r6, [r0], #0x08		/* LD:68-6f */
586	strt	r8, [r1], #0x04		/* ST:58-5b */
587	strt	r9, [r1], #0x04		/* ST:5c-5f */
588	ldrd	r8, [r0], #0x08		/* LD:70-77 */
589	strt	r4, [r1], #0x04		/* ST:60-63 */
590	strt	r5, [r1], #0x04		/* ST:64-67 */
591	ldrd	r4, [r0], #0x08		/* LD:78-7f */
592	strt	r6, [r1], #0x04		/* ST:68-6b */
593	strt	r7, [r1], #0x04		/* ST:6c-6f */
594	strt	r8, [r1], #0x04		/* ST:70-73 */
595	strt	r9, [r1], #0x04		/* ST:74-77 */
596	subs	r2, r2, #0x80
597	strt	r4, [r1], #0x04		/* ST:78-7b */
598	strt	r5, [r1], #0x04		/* ST:7c-7f */
599	bge	.Lcopyout_w_loop128
600
601.Lcopyout_w_lessthan128:
602	adds	r2, r2, #0x80		/* Adjust for extra sub */
603	ldmeqfd	sp!, {r4-r9}
604	RETeq				/* Return now if done */
605	subs	r2, r2, #0x20
606	blt	.Lcopyout_w_lessthan32
607
608	/* Copy 32 bytes at a time */
609.Lcopyout_w_loop32:
610	ldrd	r4, [r0], #0x08
611	pld	[r0, #0x18]
612	ldrd	r6, [r0], #0x08
613	ldrd	r8, [r0], #0x08
614	strt	r4, [r1], #0x04
615	strt	r5, [r1], #0x04
616	ldrd	r4, [r0], #0x08
617	strt	r6, [r1], #0x04
618	strt	r7, [r1], #0x04
619	strt	r8, [r1], #0x04
620	strt	r9, [r1], #0x04
621	subs	r2, r2, #0x20
622	strt	r4, [r1], #0x04
623	strt	r5, [r1], #0x04
624	bge	.Lcopyout_w_loop32
625
626.Lcopyout_w_lessthan32:
627	adds	r2, r2, #0x20		/* Adjust for extra sub */
628	ldmeqfd	sp!, {r4-r9}
629	RETeq				/* Return now if done */
630
631	and	r4, r2, #0x18
632	rsb	r5, r4, #0x18
633	subs	r2, r2, r4
634	add	pc, pc, r5, lsl #1
635	nop
636
637	/* At least 24 bytes remaining */
638	ldrd	r4, [r0], #0x08
639	strt	r4, [r1], #0x04
640	strt	r5, [r1], #0x04
641	nop
642
643	/* At least 16 bytes remaining */
644	ldrd	r4, [r0], #0x08
645	strt	r4, [r1], #0x04
646	strt	r5, [r1], #0x04
647	nop
648
649	/* At least 8 bytes remaining */
650	ldrd	r4, [r0], #0x08
651	strt	r4, [r1], #0x04
652	strt	r5, [r1], #0x04
653	nop
654
655	/* Less than 8 bytes remaining */
656	ldmfd	sp!, {r4-r9}
657	RETeq				/* Return now if done */
658	mov	r3, #0x00
659
660.Lcopyout_w_less_than8:
661	subs	r2, r2, #0x04
662	ldrge	ip, [r0], #0x04
663	strget	ip, [r1], #0x04
664	RETeq				/* Return now if done */
665	addlt	r2, r2, #0x04
666	ldrb	ip, [r0], #0x01
667	cmp	r2, #0x02
668	ldrgeb	r2, [r0], #0x01
669	strbt	ip, [r1], #0x01
670	ldrgtb	ip, [r0]
671	strgebt	r2, [r1], #0x01
672	strgtbt	ip, [r1]
673	RET
674
675/*
676 * At this point, it has not been possible to word align both buffers.
677 * The destination buffer (r1) is word aligned, but the source buffer
678 * (r0) is not.
679 */
680.Lcopyout_bad_align:
681	stmfd	sp!, {r4-r7}
682	mov	r3, #0x01
683	bic	r0, r0, #0x03
684	cmp	ip, #2
685	ldr	ip, [r0], #0x04
686	bgt	.Lcopyout_bad3
687	beq	.Lcopyout_bad2
688	b	.Lcopyout_bad1
689
690.Lcopyout_bad1_loop16:
691#ifdef	__ARMEB__
692	mov	r4, ip, lsl #8
693#else
694	mov	r4, ip, lsr #8
695#endif
696	ldr	r5, [r0], #0x04
697	pld	[r0, #0x018]
698	ldr	r6, [r0], #0x04
699	ldr	r7, [r0], #0x04
700	ldr	ip, [r0], #0x04
701#ifdef	__ARMEB__
702	orr	r4, r4, r5, lsr #24
703	mov	r5, r5, lsl #8
704	orr	r5, r5, r6, lsr #24
705	mov	r6, r6, lsl #8
706	orr	r6, r6, r7, lsr #24
707	mov	r7, r7, lsl #8
708	orr	r7, r7, ip, lsr #24
709#else
710	orr	r4, r4, r5, lsl #24
711	mov	r5, r5, lsr #8
712	orr	r5, r5, r6, lsl #24
713	mov	r6, r6, lsr #8
714	orr	r6, r6, r7, lsl #24
715	mov	r7, r7, lsr #8
716	orr	r7, r7, ip, lsl #24
717#endif
718	strt	r4, [r1], #0x04
719	strt	r5, [r1], #0x04
720	strt	r6, [r1], #0x04
721	strt	r7, [r1], #0x04
722.Lcopyout_bad1:
723	subs	r2, r2, #0x10
724	bge	.Lcopyout_bad1_loop16
725
726	adds	r2, r2, #0x10
727	ldmeqfd	sp!, {r4-r7}
728	RETeq				/* Return now if done */
729	subs	r2, r2, #0x04
730	sublt	r0, r0, #0x03
731	blt	.Lcopyout_l4
732
733.Lcopyout_bad1_loop4:
734#ifdef __ARMEB__
735	mov	r4, ip, lsl #8
736#else
737	mov	r4, ip, lsr #8
738#endif
739	ldr	ip, [r0], #0x04
740	subs	r2, r2, #0x04
741#ifdef __ARMEB__
742	orr	r4, r4, ip, lsr #24
743#else
744	orr	r4, r4, ip, lsl #24
745#endif
746	strt	r4, [r1], #0x04
747	bge	.Lcopyout_bad1_loop4
748	sub	r0, r0, #0x03
749	b	.Lcopyout_l4
750
751.Lcopyout_bad2_loop16:
752#ifdef __ARMEB__
753	mov	r4, ip, lsl #16
754#else
755	mov	r4, ip, lsr #16
756#endif
757	ldr	r5, [r0], #0x04
758	pld	[r0, #0x018]
759	ldr	r6, [r0], #0x04
760	ldr	r7, [r0], #0x04
761	ldr	ip, [r0], #0x04
762#ifdef __ARMEB__
763	orr	r4, r4, r5, lsr #16
764	mov	r5, r5, lsl #16
765	orr	r5, r5, r6, lsr #16
766	mov	r6, r6, lsl #16
767	orr	r6, r6, r7, lsr #16
768	mov	r7, r7, lsl #16
769	orr	r7, r7, ip, lsr #16
770#else
771	orr	r4, r4, r5, lsl #16
772	mov	r5, r5, lsr #16
773	orr	r5, r5, r6, lsl #16
774	mov	r6, r6, lsr #16
775	orr	r6, r6, r7, lsl #16
776	mov	r7, r7, lsr #16
777	orr	r7, r7, ip, lsl #16
778#endif
779	strt	r4, [r1], #0x04
780	strt	r5, [r1], #0x04
781	strt	r6, [r1], #0x04
782	strt	r7, [r1], #0x04
783.Lcopyout_bad2:
784	subs	r2, r2, #0x10
785	bge	.Lcopyout_bad2_loop16
786
787	adds	r2, r2, #0x10
788	ldmeqfd	sp!, {r4-r7}
789	RETeq				/* Return now if done */
790	subs	r2, r2, #0x04
791	sublt	r0, r0, #0x02
792	blt	.Lcopyout_l4
793
794.Lcopyout_bad2_loop4:
795#ifdef __ARMEB__
796	mov	r4, ip, lsl #16
797#else
798	mov	r4, ip, lsr #16
799#endif
800	ldr	ip, [r0], #0x04
801	subs	r2, r2, #0x04
802#ifdef __ARMEB__
803	orr	r4, r4, ip, lsr #16
804#else
805	orr	r4, r4, ip, lsl #16
806#endif
807	strt	r4, [r1], #0x04
808	bge	.Lcopyout_bad2_loop4
809	sub	r0, r0, #0x02
810	b	.Lcopyout_l4
811
812.Lcopyout_bad3_loop16:
813#ifdef __ARMEB__
814	mov	r4, ip, lsl #24
815#else
816	mov	r4, ip, lsr #24
817#endif
818	ldr	r5, [r0], #0x04
819	pld	[r0, #0x018]
820	ldr	r6, [r0], #0x04
821	ldr	r7, [r0], #0x04
822	ldr	ip, [r0], #0x04
823#ifdef __ARMEB__
824	orr	r4, r4, r5, lsr #8
825	mov	r5, r5, lsl #24
826	orr	r5, r5, r6, lsr #8
827	mov	r6, r6, lsl #24
828	orr	r6, r6, r7, lsr #8
829	mov	r7, r7, lsl #24
830	orr	r7, r7, ip, lsr #8
831#else
832	orr	r4, r4, r5, lsl #8
833	mov	r5, r5, lsr #24
834	orr	r5, r5, r6, lsl #8
835	mov	r6, r6, lsr #24
836	orr	r6, r6, r7, lsl #8
837	mov	r7, r7, lsr #24
838	orr	r7, r7, ip, lsl #8
839#endif
840	strt	r4, [r1], #0x04
841	strt	r5, [r1], #0x04
842	strt	r6, [r1], #0x04
843	strt	r7, [r1], #0x04
844.Lcopyout_bad3:
845	subs	r2, r2, #0x10
846	bge	.Lcopyout_bad3_loop16
847
848	adds	r2, r2, #0x10
849	ldmeqfd	sp!, {r4-r7}
850	RETeq				/* Return now if done */
851	subs	r2, r2, #0x04
852	sublt	r0, r0, #0x01
853	blt	.Lcopyout_l4
854
855.Lcopyout_bad3_loop4:
856#ifdef __ARMEB__
857	mov	r4, ip, lsl #24
858#else
859	mov	r4, ip, lsr #24
860#endif
861	ldr	ip, [r0], #0x04
862	subs	r2, r2, #0x04
863#ifdef __ARMEB__
864	orr	r4, r4, ip, lsr #8
865#else
866	orr	r4, r4, ip, lsl #8
867#endif
868	strt	r4, [r1], #0x04
869	bge	.Lcopyout_bad3_loop4
870	sub	r0, r0, #0x01
871
872.Lcopyout_l4:
873	ldmfd	sp!, {r4-r7}
874	mov	r3, #0x00
875	adds	r2, r2, #0x04
876	RETeq
877.Lcopyout_l4_2:
878	rsbs	r2, r2, #0x03
879	addne	pc, pc, r2, lsl #3
880	nop
881	ldrb	ip, [r0], #0x01
882	strbt	ip, [r1], #0x01
883	ldrb	ip, [r0], #0x01
884	strbt	ip, [r1], #0x01
885	ldrb	ip, [r0]
886	strbt	ip, [r1]
887	RET
888
889/*
890 * r0 - user space address
891 * r1 - kernel space address
892 * r2 - maxlens
893 * r3 - lencopied
894 *
895 * Copy string from user space to kernel space
896 */
897
898/*
899 * r0 - from
900 * r1 - to
901 * r2 - maxlens
902 * r3 - lencopied
903 *
904 * Copy string from r0 to r1
905 */
906EnterARM(copystr)
907	stmfd	sp!, {r4-r5}			/* stack is 8 byte aligned */
908	teq	r2, #0x00000000
909	mov	r5, #0x00000000
910	moveq	r0, #ENAMETOOLONG
911	beq	2f
912
9131:	ldrb	r4, [r0], #0x0001
914	add	r5, r5, #0x00000001
915	teq	r4, #0x00000000
916	strb	r4, [r1], #0x0001
917	teqne	r5, r2
918	bne	1b
919
920	teq	r4, #0x00000000
921	moveq	r0, #0x00000000
922	movne	r0, #ENAMETOOLONG
923
9242:	teq	r3, #0x00000000
925	strne	r5, [r3]
926
927	ldmfd	sp!, {r4-r5}			/* stack is 8 byte aligned */
928	mov	pc, lr
929
930
931/*
932 * r0 - user space address
933 * r1 - kernel space address
934 * r2 - maxlens
935 * r3 - lencopied
936 *
937 * Copy string from user space to kernel space
938 */
939EnterARM(copyinstr)
940	SAVE_REGS
941
942	teq	r2, #0x00000000
943	mov	r6, #0x00000000
944	moveq	r0, #ENAMETOOLONG
945	beq	2f
946
947   LoadThreadRegister(r4)
948
949	adr	r5, .Lcopystrfault
950	str	r5, [r4, TH_RECOVER]
951
9521:	ldrbt	r5, [r0], #0x0001
953	add	r6, r6, #0x00000001
954	teq	r5, #0x00000000
955	strb	r5, [r1], #0x0001
956	teqne	r6, r2
957	bne	1b
958
959	mov	r0, #0x00000000
960	str	r0, [r4, TH_RECOVER]
961
962	teq	r5, #0x00000000
963	moveq	r0, #0x00000000
964	movne	r0, #ENAMETOOLONG
965
9662:	teq	r3, #0x00000000
967	strne	r6, [r3]
968
969	RESTORE_REGS
970	bx      lr
971
972/*
973 * r0 - kernel space address
974 * r1 - user space address
975 * r2 - maxlens
976 * r3 - lencopied
977 *
978 * Copy string from kernel space to user space
979 */
980EnterARM(copyoutstr)
981	SAVE_REGS
982
983	teq	r2, #0x00000000
984	mov	r6, #0x00000000
985	moveq	r0, #ENAMETOOLONG
986	beq	2f
987
988    LoadThreadRegister(r4)
989
990	adr	r5, .Lcopystrfault
991	str	r5, [r4, TH_RECOVER]
992
9931:	ldrb	r5, [r0], #0x0001
994	add	r6, r6, #0x00000001
995	teq	r5, #0x00000000
996	strbt	r5, [r1], #0x0001
997	teqne	r6, r2
998	bne	1b
999
1000	mov	r0, #0x00000000
1001	str	r0, [r4, TH_RECOVER]
1002
1003	teq	r5, #0x00000000
1004	moveq	r0, #0x00000000
1005	movne	r0, #ENAMETOOLONG
1006
10072:	teq	r3, #0x00000000
1008	strne	r6, [r3]
1009
1010	RESTORE_REGS
1011	bx      lr
1012
1013/* A fault occurred during the copy */
1014.Lcopystrfault:
1015	mov	r1, #0x00000000
1016	str	r1, [r4, TH_RECOVER]
1017	RESTORE_REGS
1018    bx      lr
1019
1020copyio_kernel:
1021	/* We're open to attack here since we don't really verify the addresses being copied... */
1022    cmp     r2, #0x10
1023    blt     .Lcopyio_copy2
1024    orr     r3, r0, r1
1025    tst     r3, #3
1026    bne     .Lcopyio_copy2
1027    sub     r2, r2, #8
1028.Lcopyio_copy8_loop:
1029    ldr     r3, [r0], #4
1030    ldr     r12, [r0], #4
1031    str     r3, [r1], #4
1032    str     r12, [r1], #4
1033    subs    r2, r2, #8
1034    bge     .Lcopyio_copy8_loop
1035    adds    r2, r2, #8
1036    beq 	.Lcopyio_exit
1037.Lcopyio_copy2:
1038    subs    r2, r2, #2
1039    ldrb    r3, [r0], #1
1040    ldrplb  r12, [r0], #1
1041    strb    r3, [r0], #1
1042    strplb  r12, [r1], #1
1043    bhi     .Lcopyio_copy2
1044.Lcopyio_exit:
1045    movs    r0, #0
1046    bx      lr
1047copyio_no_perms:
1048    mov     r0, #0xE
1049    bx      lr
1050
1051
1052/**
1053 * copyinframe
1054 *
1055 * Safely copy in a fp and lr.
1056 */
1057EnterARM(copyinframe)
1058	stmfd	sp!,{r4}
1059	adr 	r3, _copyinframe_error
1060	LoadThreadRegister(r12)
1061	ldr 	r4, [r12, TH_RECOVER]
1062	str 	r3, [r12, TH_RECOVER]
1063	ldmia	r0, {r2,r3}
1064	stmia 	r1, {r2,r3}
1065_copyinframe_success:
1066	mov 	r0, #0
1067	LoadThreadRegister(r12)
1068	ldmfd 	sp!, {r4}
1069	bx 		lr
1070_copyinframe_error:
1071	mov 	r0, #0xE
1072	bx 		lr
1073
1074LOAD_ADDR_GEN_DEF(kernel_pmap)
1075
1076