1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2013 ARM Ltd.
4 * Copyright (C) 2013 Linaro.
5 *
6 * This code is based on glibc cortex strings work originally authored by Linaro
7 * be found @
8 *
9 * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10 * files/head:/src/aarch64/
11 */
12
13
14/*
15 * Copy a buffer from src to dest (alignment handled by the hardware)
16 *
17 * Parameters:
18 *	x0 - dest
19 *	x1 - src
20 *	x2 - n
21 * Returns:
22 *	x0 - dest
23 */
24dstin	.req	x0
25src	.req	x1
26count	.req	x2
27tmp1	.req	x3
28tmp1w	.req	w3
29tmp2	.req	x4
30tmp2w	.req	w4
31dst	.req	x6
32
33A_l	.req	x7
34A_h	.req	x8
35B_l	.req	x9
36B_h	.req	x10
37C_l	.req	x11
38C_h	.req	x12
39D_l	.req	x13
40D_h	.req	x14
41
42	mov	dst, dstin
43	cmp	count, #16
44	/*When memory length is less than 16, the accessed are not aligned.*/
45	b.lo	.Ltiny15
46
47	neg	tmp2, src
48	ands	tmp2, tmp2, #15/* Bytes to reach alignment. */
49	b.eq	.LSrcAligned
50	sub	count, count, tmp2
51	/*
52	* Copy the leading memory data from src to dst in an increasing
53	* address order.By this way,the risk of overwriting the source
54	* memory data is eliminated when the distance between src and
55	* dst is less than 16. The memory accesses here are alignment.
56	*/
57	tbz	tmp2, #0, 1f
58	ldrb1	tmp1w, src, #1
59	strb1	tmp1w, dst, #1
601:
61	tbz	tmp2, #1, 2f
62	ldrh1	tmp1w, src, #2
63	strh1	tmp1w, dst, #2
642:
65	tbz	tmp2, #2, 3f
66	ldr1	tmp1w, src, #4
67	str1	tmp1w, dst, #4
683:
69	tbz	tmp2, #3, .LSrcAligned
70	ldr1	tmp1, src, #8
71	str1	tmp1, dst, #8
72
73.LSrcAligned:
74	cmp	count, #64
75	b.ge	.Lcpy_over64
76	/*
77	* Deal with small copies quickly by dropping straight into the
78	* exit block.
79	*/
80.Ltail63:
81	/*
82	* Copy up to 48 bytes of data. At this point we only need the
83	* bottom 6 bits of count to be accurate.
84	*/
85	ands	tmp1, count, #0x30
86	b.eq	.Ltiny15
87	cmp	tmp1w, #0x20
88	b.eq	1f
89	b.lt	2f
90	ldp1	A_l, A_h, src, #16
91	stp1	A_l, A_h, dst, #16
921:
93	ldp1	A_l, A_h, src, #16
94	stp1	A_l, A_h, dst, #16
952:
96	ldp1	A_l, A_h, src, #16
97	stp1	A_l, A_h, dst, #16
98.Ltiny15:
99	/*
100	* Prefer to break one ldp/stp into several load/store to access
101	* memory in an increasing address order,rather than to load/store 16
102	* bytes from (src-16) to (dst-16) and to backward the src to aligned
103	* address,which way is used in original cortex memcpy. If keeping
104	* the original memcpy process here, memmove need to satisfy the
105	* precondition that src address is at least 16 bytes bigger than dst
106	* address,otherwise some source data will be overwritten when memove
107	* call memcpy directly. To make memmove simpler and decouple the
108	* memcpy's dependency on memmove, withdrew the original process.
109	*/
110	tbz	count, #3, 1f
111	ldr1	tmp1, src, #8
112	str1	tmp1, dst, #8
1131:
114	tbz	count, #2, 2f
115	ldr1	tmp1w, src, #4
116	str1	tmp1w, dst, #4
1172:
118	tbz	count, #1, 3f
119	ldrh1	tmp1w, src, #2
120	strh1	tmp1w, dst, #2
1213:
122	tbz	count, #0, .Lexitfunc
123	ldrb1	tmp1w, src, #1
124	strb1	tmp1w, dst, #1
125
126	b	.Lexitfunc
127
128.Lcpy_over64:
129	subs	count, count, #128
130	b.ge	.Lcpy_body_large
131	/*
132	* Less than 128 bytes to copy, so handle 64 here and then jump
133	* to the tail.
134	*/
135	ldp1	A_l, A_h, src, #16
136	stp1	A_l, A_h, dst, #16
137	ldp1	B_l, B_h, src, #16
138	ldp1	C_l, C_h, src, #16
139	stp1	B_l, B_h, dst, #16
140	stp1	C_l, C_h, dst, #16
141	ldp1	D_l, D_h, src, #16
142	stp1	D_l, D_h, dst, #16
143
144	tst	count, #0x3f
145	b.ne	.Ltail63
146	b	.Lexitfunc
147
148	/*
149	* Critical loop.  Start at a new cache line boundary.  Assuming
150	* 64 bytes per line this ensures the entire loop is in one line.
151	*/
152	.p2align	L1_CACHE_SHIFT
153.Lcpy_body_large:
154	/* pre-get 64 bytes data. */
155	ldp1	A_l, A_h, src, #16
156	ldp1	B_l, B_h, src, #16
157	ldp1	C_l, C_h, src, #16
158	ldp1	D_l, D_h, src, #16
1591:
160	/*
161	* interlace the load of next 64 bytes data block with store of the last
162	* loaded 64 bytes data.
163	*/
164	stp1	A_l, A_h, dst, #16
165	ldp1	A_l, A_h, src, #16
166	stp1	B_l, B_h, dst, #16
167	ldp1	B_l, B_h, src, #16
168	stp1	C_l, C_h, dst, #16
169	ldp1	C_l, C_h, src, #16
170	stp1	D_l, D_h, dst, #16
171	ldp1	D_l, D_h, src, #16
172	subs	count, count, #64
173	b.ge	1b
174	stp1	A_l, A_h, dst, #16
175	stp1	B_l, B_h, dst, #16
176	stp1	C_l, C_h, dst, #16
177	stp1	D_l, D_h, dst, #16
178
179	tst	count, #0x3f
180	b.ne	.Ltail63
181.Lexitfunc:
182