1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) IBM Corporation, 2011
4 * Derived from copyuser_power7.s by Anton Blanchard <anton@au.ibm.com>
5 * Author - Balbir Singh <bsingharora@gmail.com>
6 */
7#include <linux/export.h>
8#include <asm/ppc_asm.h>
9#include <asm/errno.h>
10
11	.macro err1
12100:
13	EX_TABLE(100b,.Ldo_err1)
14	.endm
15
16	.macro err2
17200:
18	EX_TABLE(200b,.Ldo_err2)
19	.endm
20
21	.macro err3
22300:	EX_TABLE(300b,.Ldone)
23	.endm
24
25.Ldo_err2:
26	ld	r22,STK_REG(R22)(r1)
27	ld	r21,STK_REG(R21)(r1)
28	ld	r20,STK_REG(R20)(r1)
29	ld	r19,STK_REG(R19)(r1)
30	ld	r18,STK_REG(R18)(r1)
31	ld	r17,STK_REG(R17)(r1)
32	ld	r16,STK_REG(R16)(r1)
33	ld	r15,STK_REG(R15)(r1)
34	ld	r14,STK_REG(R14)(r1)
35	addi	r1,r1,STACKFRAMESIZE
36.Ldo_err1:
37	/* Do a byte by byte copy to get the exact remaining size */
38	mtctr	r7
3946:
40err3;	lbz	r0,0(r4)
41	addi	r4,r4,1
42err3;	stb	r0,0(r3)
43	addi	r3,r3,1
44	bdnz	46b
45	li	r3,0
46	blr
47
48.Ldone:
49	mfctr	r3
50	blr
51
52
53_GLOBAL(copy_mc_generic)
54	mr	r7,r5
55	cmpldi	r5,16
56	blt	.Lshort_copy
57
58.Lcopy:
59	/* Get the source 8B aligned */
60	neg	r6,r4
61	mtocrf	0x01,r6
62	clrldi	r6,r6,(64-3)
63
64	bf	cr7*4+3,1f
65err1;	lbz	r0,0(r4)
66	addi	r4,r4,1
67err1;	stb	r0,0(r3)
68	addi	r3,r3,1
69	subi	r7,r7,1
70
711:	bf	cr7*4+2,2f
72err1;	lhz	r0,0(r4)
73	addi	r4,r4,2
74err1;	sth	r0,0(r3)
75	addi	r3,r3,2
76	subi	r7,r7,2
77
782:	bf	cr7*4+1,3f
79err1;	lwz	r0,0(r4)
80	addi	r4,r4,4
81err1;	stw	r0,0(r3)
82	addi	r3,r3,4
83	subi	r7,r7,4
84
853:	sub	r5,r5,r6
86	cmpldi	r5,128
87
88	mflr	r0
89	stdu	r1,-STACKFRAMESIZE(r1)
90	std	r14,STK_REG(R14)(r1)
91	std	r15,STK_REG(R15)(r1)
92	std	r16,STK_REG(R16)(r1)
93	std	r17,STK_REG(R17)(r1)
94	std	r18,STK_REG(R18)(r1)
95	std	r19,STK_REG(R19)(r1)
96	std	r20,STK_REG(R20)(r1)
97	std	r21,STK_REG(R21)(r1)
98	std	r22,STK_REG(R22)(r1)
99	std	r0,STACKFRAMESIZE+16(r1)
100
101	blt	5f
102	srdi	r6,r5,7
103	mtctr	r6
104
105	/* Now do cacheline (128B) sized loads and stores. */
106	.align	5
1074:
108err2;	ld	r0,0(r4)
109err2;	ld	r6,8(r4)
110err2;	ld	r8,16(r4)
111err2;	ld	r9,24(r4)
112err2;	ld	r10,32(r4)
113err2;	ld	r11,40(r4)
114err2;	ld	r12,48(r4)
115err2;	ld	r14,56(r4)
116err2;	ld	r15,64(r4)
117err2;	ld	r16,72(r4)
118err2;	ld	r17,80(r4)
119err2;	ld	r18,88(r4)
120err2;	ld	r19,96(r4)
121err2;	ld	r20,104(r4)
122err2;	ld	r21,112(r4)
123err2;	ld	r22,120(r4)
124	addi	r4,r4,128
125err2;	std	r0,0(r3)
126err2;	std	r6,8(r3)
127err2;	std	r8,16(r3)
128err2;	std	r9,24(r3)
129err2;	std	r10,32(r3)
130err2;	std	r11,40(r3)
131err2;	std	r12,48(r3)
132err2;	std	r14,56(r3)
133err2;	std	r15,64(r3)
134err2;	std	r16,72(r3)
135err2;	std	r17,80(r3)
136err2;	std	r18,88(r3)
137err2;	std	r19,96(r3)
138err2;	std	r20,104(r3)
139err2;	std	r21,112(r3)
140err2;	std	r22,120(r3)
141	addi	r3,r3,128
142	subi	r7,r7,128
143	bdnz	4b
144
145	clrldi	r5,r5,(64-7)
146
147	/* Up to 127B to go */
1485:	srdi	r6,r5,4
149	mtocrf	0x01,r6
150
1516:	bf	cr7*4+1,7f
152err2;	ld	r0,0(r4)
153err2;	ld	r6,8(r4)
154err2;	ld	r8,16(r4)
155err2;	ld	r9,24(r4)
156err2;	ld	r10,32(r4)
157err2;	ld	r11,40(r4)
158err2;	ld	r12,48(r4)
159err2;	ld	r14,56(r4)
160	addi	r4,r4,64
161err2;	std	r0,0(r3)
162err2;	std	r6,8(r3)
163err2;	std	r8,16(r3)
164err2;	std	r9,24(r3)
165err2;	std	r10,32(r3)
166err2;	std	r11,40(r3)
167err2;	std	r12,48(r3)
168err2;	std	r14,56(r3)
169	addi	r3,r3,64
170	subi	r7,r7,64
171
1727:	ld	r14,STK_REG(R14)(r1)
173	ld	r15,STK_REG(R15)(r1)
174	ld	r16,STK_REG(R16)(r1)
175	ld	r17,STK_REG(R17)(r1)
176	ld	r18,STK_REG(R18)(r1)
177	ld	r19,STK_REG(R19)(r1)
178	ld	r20,STK_REG(R20)(r1)
179	ld	r21,STK_REG(R21)(r1)
180	ld	r22,STK_REG(R22)(r1)
181	addi	r1,r1,STACKFRAMESIZE
182
183	/* Up to 63B to go */
184	bf	cr7*4+2,8f
185err1;	ld	r0,0(r4)
186err1;	ld	r6,8(r4)
187err1;	ld	r8,16(r4)
188err1;	ld	r9,24(r4)
189	addi	r4,r4,32
190err1;	std	r0,0(r3)
191err1;	std	r6,8(r3)
192err1;	std	r8,16(r3)
193err1;	std	r9,24(r3)
194	addi	r3,r3,32
195	subi	r7,r7,32
196
197	/* Up to 31B to go */
1988:	bf	cr7*4+3,9f
199err1;	ld	r0,0(r4)
200err1;	ld	r6,8(r4)
201	addi	r4,r4,16
202err1;	std	r0,0(r3)
203err1;	std	r6,8(r3)
204	addi	r3,r3,16
205	subi	r7,r7,16
206
2079:	clrldi	r5,r5,(64-4)
208
209	/* Up to 15B to go */
210.Lshort_copy:
211	mtocrf	0x01,r5
212	bf	cr7*4+0,12f
213err1;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
214err1;	lwz	r6,4(r4)
215	addi	r4,r4,8
216err1;	stw	r0,0(r3)
217err1;	stw	r6,4(r3)
218	addi	r3,r3,8
219	subi	r7,r7,8
220
22112:	bf	cr7*4+1,13f
222err1;	lwz	r0,0(r4)
223	addi	r4,r4,4
224err1;	stw	r0,0(r3)
225	addi	r3,r3,4
226	subi	r7,r7,4
227
22813:	bf	cr7*4+2,14f
229err1;	lhz	r0,0(r4)
230	addi	r4,r4,2
231err1;	sth	r0,0(r3)
232	addi	r3,r3,2
233	subi	r7,r7,2
234
23514:	bf	cr7*4+3,15f
236err1;	lbz	r0,0(r4)
237err1;	stb	r0,0(r3)
238
23915:	li	r3,0
240	blr
241
242EXPORT_SYMBOL_GPL(copy_mc_generic);
243