copyinout.S revision 319202 
1/*-
2 * Copyright (c) 2015 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Andrew Turner under
6 * sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 */
30
31#include <machine/asm.h>
32__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/copyinout.S 319202 2017-05-30 12:26:36Z andrew $");
33
34#include <sys/errno.h>
35
36#include <machine/vmparam.h>
37
38#include "assym.s"
39
40/*
41 * Fault handler for the copy{in,out} functions below.
42 */
43ENTRY(copyio_fault)
44	SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */
45	EXIT_USER_ACCESS_CHECK(w0, x1)
46copyio_fault_nopcb:
47	mov	x0, #EFAULT
48	ret
49END(copyio_fault)
50
51/*
52 * Copies from a kernel to user address
53 *
54 * int copyout(const void *kaddr, void *udaddr, size_t len)
55 */
56ENTRY(copyout)
57	cbz	x2, 1f
58	adds	x3, x1, x2
59	b.cs 	copyio_fault_nopcb
60	ldr	x4, =VM_MAXUSER_ADDRESS
61	cmp	x3, x4
62	b.hi	copyio_fault_nopcb
63
64	b	copycommon
65
661:	mov	x0, xzr		/* return 0 */
67	ret
68
69END(copyout)
70
71/*
72 * Copies from a user to kernel address
73 *
74 * int copyin(const void *uaddr, void *kdaddr, size_t len)
75 */
76ENTRY(copyin)
77	cbz	x2, 1f
78	adds	x3, x0, x2
79	b.cs    copyio_fault_nopcb
80	ldr	x4, =VM_MAXUSER_ADDRESS
81	cmp	x3, x4
82	b.hi	copyio_fault_nopcb
83
84	b	copycommon
85
861:	mov	x0, xzr		/* return 0 */
87	ret
88
89END(copyin)
90
91/*
92 * Copies a string from a user to kernel address
93 *
94 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
95 */
96ENTRY(copyinstr)
97	mov	x5, xzr		/* count = 0 */
98	mov	w4, #1		/* If zero return faulure */
99	cbz	x2, 3f		/* If len == 0 then skip loop */
100
101	adr	x6, copyio_fault /* Get the handler address */
102	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
103	ENTER_USER_ACCESS(w6, x7)
104
105	ldr	x7, =VM_MAXUSER_ADDRESS
1061:	cmp	x0, x7
107	b.cs	copyio_fault
108	ldtrb	w4, [x0]	/* Load from uaddr */
109	add	x0, x0, #1	/* Next char */
110	strb	w4, [x1], #1	/* Store in kaddr */
111	add	x5, x5, #1	/* count++ */
112	cbz	w4, 2f		/* Break when NUL-terminated */
113	sub	x2, x2, #1	/* len-- */
114	cbnz	x2, 1b
115
1162:	EXIT_USER_ACCESS(w6)
117	SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
118
119
1203:	cbz	x3, 4f		/* Check if done != NULL */
121	str	x5, [x3]	/* done = count */
122
1234:	mov	w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */
124	cmp	w4, #0		/* Check if we saved the NUL-terminator */
125	csel	w0, wzr, w1, eq	/* If so return success, else failure */
126	ret
127END(copyinstr)
128
129/*
130 * Local helper
131 *
132 * x0 - src pointer
133 * x1 - dst pointer
134 * x2 - size
135 * lr - the return address, so jump here instead of calling
136 *
137 * This function is optimized to minimize concurrent memory accesses. In
138 * present form it is suited for cores with a single memory prefetching
139 * unit.
140 * ARM64TODO:
141 *   Consider using separate functions for each ARM64 core. Adding memory
142 *   access interleaving might increase a total throughput on A57 or A72.
143 */
144	.text
145	.align	4
146	.local	copycommon
147	.type	copycommon,@function
148
149copycommon:
150	adr	x6, copyio_fault /* Get the handler address */
151	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
152	ENTER_USER_ACCESS(w6, x7)
153
154	/* Check alignment */
155	orr	x3, x0, x1
156	ands	x3, x3, 0x07
157	b.eq	aligned
158
159	/* Unaligned is byte by byte copy */
160byte_by_byte:
161	ldrb	w3, [x0], #0x01
162	strb	w3, [x1], #0x01
163	subs	x2, x2, #0x01
164	b.ne	byte_by_byte
165	b	ending
166
167aligned:
168	cmp	x2, #0x10
169	b.lt	lead_out
170	cmp	x2, #0x40
171	b.lt	by_dwords_start
172
173	/* Block copy */
174	lsr	x15, x2, #0x06
175by_blocks:
176	ldp	x3, x4, [x0], #0x10
177	ldp	x5, x6, [x0], #0x10
178	ldp	x7, x8, [x0], #0x10
179	ldp	x9, x10, [x0], #0x10
180	stp	x3, x4, [x1], #0x10
181	stp	x5, x6, [x1], #0x10
182	stp	x7, x8, [x1], #0x10
183	stp	x9, x10, [x1], #0x10
184
185	subs	x15, x15, #0x01
186	b.ne	by_blocks
187
188	and	x2, x2, #0x3f
189
190by_dwords_start:
191	lsr	x15, x2, #0x04
192	cbz	x15, lead_out
193by_dwords:
194	ldp	x3, x4, [x0], #0x10
195	stp	x3, x4, [x1], #0x10
196	subs	x15, x15, #0x01
197	b.ne  	by_dwords
198
199	/* Less than 16 bytes to copy */
200lead_out:
201	tbz	x2, #0x03, last_word
202	ldr	x3, [x0], #0x08
203	str	x3, [x1], #0x08
204
205last_word:
206	tbz	x2, #0x02, last_hword
207	ldr	w3, [x0], #0x04
208	str	w3, [x1], #0x04
209
210last_hword:
211	tbz	x2, #0x01, last_byte
212	ldrh	w3, [x0], #0x02
213	strh	w3, [x1], #0x02
214
215last_byte:
216	tbz	x2, #0x00, ending
217	ldrb	w3, [x0]
218	strb	w3, [x1]
219
220ending:
221	EXIT_USER_ACCESS_CHECK(w6, x7)
222	SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
223
224	mov	x0, xzr		/* return 0 */
225	ret
226	.size	copycommon, . - copycommon
227