1/*
2 * Copyright 2002, 2003 Andi Kleen, SuSE Labs.
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License.  See the file COPYING in the main directory of this archive
6 * for more details. No warranty for anything given at all.
7 */
8#include <linux/linkage.h>
9#include <asm/errno.h>
10#include <asm/asm.h>
11
12/*
13 * Checksum copy with exception handling.
14 * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
15 * destination is zeroed.
16 *
17 * Input
18 * rdi  source
19 * rsi  destination
20 * edx  len (32bit)
21 *
22 * Output
23 * eax  64bit sum. undefined in case of exception.
24 *
25 * Wrappers need to take care of valid exception sum and zeroing.
26 * They also should align source or destination to 8 bytes.
27 */
28
29	.macro source
3010:
31	_ASM_EXTABLE_UA(10b, .Lfault)
32	.endm
33
34	.macro dest
3520:
36	_ASM_EXTABLE_UA(20b, .Lfault)
37	.endm
38
39SYM_FUNC_START(csum_partial_copy_generic)
40	subq  $5*8, %rsp
41	movq  %rbx, 0*8(%rsp)
42	movq  %r12, 1*8(%rsp)
43	movq  %r14, 2*8(%rsp)
44	movq  %r13, 3*8(%rsp)
45	movq  %r15, 4*8(%rsp)
46
47	movl  $-1, %eax
48	xorl  %r9d, %r9d
49	movl  %edx, %ecx
50	cmpl  $8, %ecx
51	jb    .Lshort
52
53	testb  $7, %sil
54	jne   .Lunaligned
55.Laligned:
56	movl  %ecx, %r12d
57
58	shrq  $6, %r12
59	jz	.Lhandle_tail       /* < 64 */
60
61	clc
62
63	/* main loop. clear in 64 byte blocks */
64	/* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
65	/* r11:	temp3, rdx: temp4, r12 loopcnt */
66	/* r10:	temp5, r15: temp6, r14 temp7, r13 temp8 */
67	.p2align 4
68.Lloop:
69	source
70	movq  (%rdi), %rbx
71	source
72	movq  8(%rdi), %r8
73	source
74	movq  16(%rdi), %r11
75	source
76	movq  24(%rdi), %rdx
77
78	source
79	movq  32(%rdi), %r10
80	source
81	movq  40(%rdi), %r15
82	source
83	movq  48(%rdi), %r14
84	source
85	movq  56(%rdi), %r13
86
8730:
88	/*
89	 * No _ASM_EXTABLE_UA; this is used for intentional prefetch on a
90	 * potentially unmapped kernel address.
91	 */
92	_ASM_EXTABLE(30b, 2f)
93	prefetcht0 5*64(%rdi)
942:
95	adcq  %rbx, %rax
96	adcq  %r8, %rax
97	adcq  %r11, %rax
98	adcq  %rdx, %rax
99	adcq  %r10, %rax
100	adcq  %r15, %rax
101	adcq  %r14, %rax
102	adcq  %r13, %rax
103
104	decl %r12d
105
106	dest
107	movq %rbx, (%rsi)
108	dest
109	movq %r8, 8(%rsi)
110	dest
111	movq %r11, 16(%rsi)
112	dest
113	movq %rdx, 24(%rsi)
114
115	dest
116	movq %r10, 32(%rsi)
117	dest
118	movq %r15, 40(%rsi)
119	dest
120	movq %r14, 48(%rsi)
121	dest
122	movq %r13, 56(%rsi)
123
124	leaq 64(%rdi), %rdi
125	leaq 64(%rsi), %rsi
126
127	jnz	.Lloop
128
129	adcq  %r9, %rax
130
131	/* do last up to 56 bytes */
132.Lhandle_tail:
133	/* ecx:	count, rcx.63: the end result needs to be rol8 */
134	movq %rcx, %r10
135	andl $63, %ecx
136	shrl $3, %ecx
137	jz	.Lfold
138	clc
139	.p2align 4
140.Lloop_8:
141	source
142	movq (%rdi), %rbx
143	adcq %rbx, %rax
144	decl %ecx
145	dest
146	movq %rbx, (%rsi)
147	leaq 8(%rsi), %rsi /* preserve carry */
148	leaq 8(%rdi), %rdi
149	jnz	.Lloop_8
150	adcq %r9, %rax	/* add in carry */
151
152.Lfold:
153	/* reduce checksum to 32bits */
154	movl %eax, %ebx
155	shrq $32, %rax
156	addl %ebx, %eax
157	adcl %r9d, %eax
158
159	/* do last up to 6 bytes */
160.Lhandle_7:
161	movl %r10d, %ecx
162	andl $7, %ecx
163.L1:				/* .Lshort rejoins the common path here */
164	shrl $1, %ecx
165	jz   .Lhandle_1
166	movl $2, %edx
167	xorl %ebx, %ebx
168	clc
169	.p2align 4
170.Lloop_1:
171	source
172	movw (%rdi), %bx
173	adcl %ebx, %eax
174	decl %ecx
175	dest
176	movw %bx, (%rsi)
177	leaq 2(%rdi), %rdi
178	leaq 2(%rsi), %rsi
179	jnz .Lloop_1
180	adcl %r9d, %eax	/* add in carry */
181
182	/* handle last odd byte */
183.Lhandle_1:
184	testb $1, %r10b
185	jz    .Lende
186	xorl  %ebx, %ebx
187	source
188	movb (%rdi), %bl
189	dest
190	movb %bl, (%rsi)
191	addl %ebx, %eax
192	adcl %r9d, %eax		/* carry */
193
194.Lende:
195	testq %r10, %r10
196	js  .Lwas_odd
197.Lout:
198	movq 0*8(%rsp), %rbx
199	movq 1*8(%rsp), %r12
200	movq 2*8(%rsp), %r14
201	movq 3*8(%rsp), %r13
202	movq 4*8(%rsp), %r15
203	addq $5*8, %rsp
204	RET
205.Lshort:
206	movl %ecx, %r10d
207	jmp  .L1
208.Lunaligned:
209	xorl %ebx, %ebx
210	testb $1, %sil
211	jne  .Lodd
2121:	testb $2, %sil
213	je   2f
214	source
215	movw (%rdi), %bx
216	dest
217	movw %bx, (%rsi)
218	leaq 2(%rdi), %rdi
219	subq $2, %rcx
220	leaq 2(%rsi), %rsi
221	addq %rbx, %rax
2222:	testb $4, %sil
223	je .Laligned
224	source
225	movl (%rdi), %ebx
226	dest
227	movl %ebx, (%rsi)
228	leaq 4(%rdi), %rdi
229	subq $4, %rcx
230	leaq 4(%rsi), %rsi
231	addq %rbx, %rax
232	jmp .Laligned
233
234.Lodd:
235	source
236	movb (%rdi), %bl
237	dest
238	movb %bl, (%rsi)
239	leaq 1(%rdi), %rdi
240	leaq 1(%rsi), %rsi
241	/* decrement, set MSB */
242	leaq -1(%rcx, %rcx), %rcx
243	rorq $1, %rcx
244	shll $8, %ebx
245	addq %rbx, %rax
246	jmp 1b
247
248.Lwas_odd:
249	roll $8, %eax
250	jmp .Lout
251
252	/* Exception: just return 0 */
253.Lfault:
254	xorl %eax, %eax
255	jmp  .Lout
256SYM_FUNC_END(csum_partial_copy_generic)
257