1/*
2 * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
3 * All rights reserved.
4 *
5 * Author: Chris G. Demetriou
6 *
7 * Permission to use, copy, modify and distribute this software and
8 * its documentation is hereby granted, provided that both the copyright
9 * notice and this permission notice appear in all copies of the
10 * software, derivative works or modified versions, and any portions
11 * thereof, and that both notices appear in supporting documentation.
12 *
13 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
14 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
15 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
16 *
17 * Carnegie Mellon requests users of this software to return to
18 *
19 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
20 *  School of Computer Science
21 *  Carnegie Mellon University
22 *  Pittsburgh PA 15213-3890
23 *
24 * any improvements or extensions that they make and grant Carnegie the
25 * rights to redistribute these changes.
26 */
27
28#include <machine/asm.h>
29
30/*
31 * Copy a bytes within the kernel's address space.  The bcopy and memmove
32 * variants handle overlapping regions, the memcpy variant does not.
33 *
34 * void* memcpy(void *to, void *from, size_t len);
35 * void* memmove(void *to, void *from, size_t len);
36 * void bcopy(void *from, void *to, size_t len);
37 */
38LEAF(memcpy,3)
39	/* Swap arguments, also saving the original `from' in v0 */
40	cmoveq  zero,a0,v0
41	cmoveq  zero,a1,a0
42	cmoveq  zero,v0,a1
43
44 	/* Check for zero length */
45	beq	a2,bcopy_done
46
47	br	bcopy_forward
48
49XLEAF(memmove,3)
50	/* Swap arguments, also saving the original `from' in v0 */
51	cmoveq  zero,a0,v0
52	cmoveq  zero,a1,a0
53	cmoveq  zero,v0,a1
54
55XLEAF(bcopy,3)
56 	/* Check for zero length */
57	beq	a2,bcopy_done
58
59	/* Check for overlap */
60	subq	a1,a0,t5
61	cmpult	t5,a2,t5
62	bne	t5,bcopy_overlap
63
64bcopy_forward:
65	/* a3 = end address */
66	addq	a0,a2,a3
67
68	/* Get the first word */
69	ldq_u	t2,0(a0)
70
71	/* Do they have the same alignment? */
72	xor	a0,a1,t0
73	and	t0,7,t0
74	and	a1,7,t1
75	bne	t0,bcopy_different_alignment
76
77	/* src & dst have same alignment */
78	beq	t1,bcopy_all_aligned
79
80	ldq_u	t3,0(a1)
81	addq	a2,t1,a2
82	mskqh	t2,a0,t2
83	mskql	t3,a0,t3
84	or	t2,t3,t2
85
86	/* Dst is 8-byte aligned */
87
88bcopy_all_aligned:
89	/* If less than 8 bytes,skip loop */
90	subq	a2,1,t0
91	and	a2,7,a2
92	bic	t0,7,t0
93	beq	t0,bcopy_samealign_lp_end
94
95bcopy_samealign_lp:
96	stq_u	t2,0(a1)
97	addq	a1,8,a1
98	ldq_u	t2,8(a0)
99	subq	t0,8,t0
100	addq	a0,8,a0
101	bne	t0,bcopy_samealign_lp
102
103bcopy_samealign_lp_end:
104	/* If we're done, exit */
105	bne	a2,bcopy_small_left
106	stq_u	t2,0(a1)
107	RET
108
109bcopy_small_left:
110	mskql	t2,a2,t4
111	ldq_u	t3,0(a1)
112	mskqh	t3,a2,t3
113	or	t4,t3,t4
114	stq_u	t4,0(a1)
115	RET
116
117bcopy_different_alignment:
118	/*
119	 * this is the fun part
120	 */
121	addq	a0,a2,a3
122	cmpule	a2,8,t0
123	bne	t0,bcopy_da_finish
124
125	beq	t1,bcopy_da_noentry
126
127	/* Do the initial partial word */
128	subq	zero,a1,t0
129	and	t0,7,t0
130	ldq_u	t3,7(a0)
131	extql	t2,a0,t2
132	extqh	t3,a0,t3
133	or	t2,t3,t5
134	insql	t5,a1,t5
135	ldq_u	t6,0(a1)
136	mskql	t6,a1,t6
137	or	t5,t6,t5
138	stq_u	t5,0(a1)
139	addq	a0,t0,a0
140	addq	a1,t0,a1
141	subq	a2,t0,a2
142	ldq_u	t2,0(a0)
143
144bcopy_da_noentry:
145	subq	a2,1,t0
146	bic	t0,7,t0
147	and	a2,7,a2
148	beq	t0,bcopy_da_finish2
149
150bcopy_da_lp:
151	ldq_u	t3,7(a0)
152	addq	a0,8,a0
153	extql	t2,a0,t4
154	extqh	t3,a0,t5
155	subq	t0,8,t0
156	or	t4,t5,t5
157	stq	t5,0(a1)
158	addq	a1,8,a1
159	beq	t0,bcopy_da_finish1
160	ldq_u	t2,7(a0)
161	addq	a0,8,a0
162	extql	t3,a0,t4
163	extqh	t2,a0,t5
164	subq	t0,8,t0
165	or	t4,t5,t5
166	stq	t5,0(a1)
167	addq	a1,8,a1
168	bne	t0,bcopy_da_lp
169
170bcopy_da_finish2:
171	/* Do the last new word */
172	mov	t2,t3
173
174bcopy_da_finish1:
175	/* Do the last partial word */
176	ldq_u	t2,-1(a3)
177	extql	t3,a0,t3
178	extqh	t2,a0,t2
179	or	t2,t3,t2
180	br	zero,bcopy_samealign_lp_end
181
182bcopy_da_finish:
183	/* Do the last word in the next source word */
184	ldq_u	t3,-1(a3)
185	extql	t2,a0,t2
186	extqh	t3,a0,t3
187	or	t2,t3,t2
188	insqh	t2,a1,t3
189	insql	t2,a1,t2
190	lda	t4,-1(zero)
191	mskql	t4,a2,t5
192	cmovne	t5,t5,t4
193	insqh	t4,a1,t5
194	insql	t4,a1,t4
195	addq	a1,a2,a4
196	ldq_u	t6,0(a1)
197	ldq_u	t7,-1(a4)
198	bic	t6,t4,t6
199	bic	t7,t5,t7
200	and	t2,t4,t2
201	and	t3,t5,t3
202	or	t2,t6,t2
203	or	t3,t7,t3
204	stq_u	t3,-1(a4)
205	stq_u	t2,0(a1)
206	RET
207
208bcopy_overlap:
209	/*
210	 * Basically equivalent to previous case, only backwards.
211	 * Not quite as highly optimized
212	 */
213	addq	a0,a2,a3
214	addq	a1,a2,a4
215
216	/* less than 8 bytes - don't worry about overlap */
217	cmpule	a2,8,t0
218	bne	t0,bcopy_ov_short
219
220	/* Possibly do a partial first word */
221	and	a4,7,t4
222	beq	t4,bcopy_ov_nostart2
223	subq	a3,t4,a3
224	subq	a4,t4,a4
225	ldq_u	t1,0(a3)
226	subq	a2,t4,a2
227	ldq_u	t2,7(a3)
228	ldq	t3,0(a4)
229	extql	t1,a3,t1
230	extqh	t2,a3,t2
231	or	t1,t2,t1
232	mskqh	t3,t4,t3
233	mskql	t1,t4,t1
234	or	t1,t3,t1
235	stq	t1,0(a4)
236
237bcopy_ov_nostart2:
238	bic	a2,7,t4
239	and	a2,7,a2
240	beq	t4,bcopy_ov_lp_end
241
242bcopy_ov_lp:
243	/* This could be more pipelined, but it doesn't seem worth it */
244	ldq_u	t0,-8(a3)
245	subq	a4,8,a4
246	ldq_u	t1,-1(a3)
247	subq	a3,8,a3
248	extql	t0,a3,t0
249	extqh	t1,a3,t1
250	subq	t4,8,t4
251	or	t0,t1,t0
252	stq	t0,0(a4)
253	bne	t4,bcopy_ov_lp
254
255bcopy_ov_lp_end:
256	beq	a2,bcopy_done
257
258	ldq_u	t0,0(a0)
259	ldq_u	t1,7(a0)
260	ldq_u	t2,0(a1)
261	extql	t0,a0,t0
262	extqh	t1,a0,t1
263	or	t0,t1,t0
264	insql	t0,a1,t0
265	mskql	t2,a1,t2
266	or	t2,t0,t2
267	stq_u	t2,0(a1)
268
269bcopy_done:
270	RET
271
272bcopy_ov_short:
273	ldq_u	t2,0(a0)
274	br	zero,bcopy_da_finish
275
276	END(memcpy)
277