1// SPDX-License-Identifier: GPL-2.0
2/*
3 * csum_partial_copy - do IP checksumming and copy
4 *
5 * (C) Copyright 1996 Linus Torvalds
6 * accelerated versions (and 21264 assembly versions ) contributed by
7 *	Rick Gorton	<rick.gorton@alpha-processor.com>
8 *
9 * Don't look at this too closely - you'll go mad. The things
10 * we do for performance..
11 */
12
13#include <linux/types.h>
14#include <linux/string.h>
15#include <linux/uaccess.h>
16#include <net/checksum.h>
17
18
19#define ldq_u(x,y) \
20__asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
21
22#define stq_u(x,y) \
23__asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
24
25#define extql(x,y,z) \
26__asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
27
28#define extqh(x,y,z) \
29__asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
30
31#define mskql(x,y,z) \
32__asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
33
34#define mskqh(x,y,z) \
35__asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
36
37#define insql(x,y,z) \
38__asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
39
40#define insqh(x,y,z) \
41__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42
43#define __get_word(insn,x,ptr)				\
44({							\
45	long __guu_err;					\
46	__asm__ __volatile__(				\
47	"1:	"#insn" %0,%2\n"			\
48	"2:\n"						\
49	EXC(1b,2b,%0,%1)				\
50		: "=r"(x), "=r"(__guu_err)		\
51		: "m"(__m(ptr)), "1"(0));		\
52	__guu_err;					\
53})
54
55static inline unsigned short from64to16(unsigned long x)
56{
57	/* Using extract instructions is a bit more efficient
58	   than the original shift/bitmask version.  */
59
60	union {
61		unsigned long	ul;
62		unsigned int	ui[2];
63		unsigned short	us[4];
64	} in_v, tmp_v, out_v;
65
66	in_v.ul = x;
67	tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
68
69	/* Since the bits of tmp_v.sh[3] are going to always be zero,
70	   we don't have to bother to add that in.  */
71	out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
72			+ (unsigned long) tmp_v.us[2];
73
74	/* Similarly, out_v.us[2] is always zero for the final add.  */
75	return out_v.us[0] + out_v.us[1];
76}
77
78
79
80/*
81 * Ok. This isn't fun, but this is the EASY case.
82 */
83static inline unsigned long
84csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
85			 long len)
86{
87	unsigned long checksum = ~0U;
88	unsigned long carry = 0;
89
90	while (len >= 0) {
91		unsigned long word;
92		if (__get_word(ldq, word, src))
93			return 0;
94		checksum += carry;
95		src++;
96		checksum += word;
97		len -= 8;
98		carry = checksum < word;
99		*dst = word;
100		dst++;
101	}
102	len += 8;
103	checksum += carry;
104	if (len) {
105		unsigned long word, tmp;
106		if (__get_word(ldq, word, src))
107			return 0;
108		tmp = *dst;
109		mskql(word, len, word);
110		checksum += word;
111		mskqh(tmp, len, tmp);
112		carry = checksum < word;
113		*dst = word | tmp;
114		checksum += carry;
115	}
116	return checksum;
117}
118
119/*
120 * This is even less fun, but this is still reasonably
121 * easy.
122 */
123static inline unsigned long
124csum_partial_cfu_dest_aligned(const unsigned long __user *src,
125			      unsigned long *dst,
126			      unsigned long soff,
127			      long len)
128{
129	unsigned long first;
130	unsigned long word, carry;
131	unsigned long lastsrc = 7+len+(unsigned long)src;
132	unsigned long checksum = ~0U;
133
134	if (__get_word(ldq_u, first,src))
135		return 0;
136	carry = 0;
137	while (len >= 0) {
138		unsigned long second;
139
140		if (__get_word(ldq_u, second, src+1))
141			return 0;
142		extql(first, soff, word);
143		len -= 8;
144		src++;
145		extqh(second, soff, first);
146		checksum += carry;
147		word |= first;
148		first = second;
149		checksum += word;
150		*dst = word;
151		dst++;
152		carry = checksum < word;
153	}
154	len += 8;
155	checksum += carry;
156	if (len) {
157		unsigned long tmp;
158		unsigned long second;
159		if (__get_word(ldq_u, second, lastsrc))
160			return 0;
161		tmp = *dst;
162		extql(first, soff, word);
163		extqh(second, soff, first);
164		word |= first;
165		mskql(word, len, word);
166		checksum += word;
167		mskqh(tmp, len, tmp);
168		carry = checksum < word;
169		*dst = word | tmp;
170		checksum += carry;
171	}
172	return checksum;
173}
174
175/*
176 * This is slightly less fun than the above..
177 */
178static inline unsigned long
179csum_partial_cfu_src_aligned(const unsigned long __user *src,
180			     unsigned long *dst,
181			     unsigned long doff,
182			     long len,
183			     unsigned long partial_dest)
184{
185	unsigned long carry = 0;
186	unsigned long word;
187	unsigned long second_dest;
188	unsigned long checksum = ~0U;
189
190	mskql(partial_dest, doff, partial_dest);
191	while (len >= 0) {
192		if (__get_word(ldq, word, src))
193			return 0;
194		len -= 8;
195		insql(word, doff, second_dest);
196		checksum += carry;
197		stq_u(partial_dest | second_dest, dst);
198		src++;
199		checksum += word;
200		insqh(word, doff, partial_dest);
201		carry = checksum < word;
202		dst++;
203	}
204	len += 8;
205	if (len) {
206		checksum += carry;
207		if (__get_word(ldq, word, src))
208			return 0;
209		mskql(word, len, word);
210		len -= 8;
211		checksum += word;
212		insql(word, doff, second_dest);
213		len += doff;
214		carry = checksum < word;
215		partial_dest |= second_dest;
216		if (len >= 0) {
217			stq_u(partial_dest, dst);
218			if (!len) goto out;
219			dst++;
220			insqh(word, doff, partial_dest);
221		}
222		doff = len;
223	}
224	ldq_u(second_dest, dst);
225	mskqh(second_dest, doff, second_dest);
226	stq_u(partial_dest | second_dest, dst);
227out:
228	checksum += carry;
229	return checksum;
230}
231
232/*
233 * This is so totally un-fun that it's frightening. Don't
234 * look at this too closely, you'll go blind.
235 */
236static inline unsigned long
237csum_partial_cfu_unaligned(const unsigned long __user * src,
238			   unsigned long * dst,
239			   unsigned long soff, unsigned long doff,
240			   long len, unsigned long partial_dest)
241{
242	unsigned long carry = 0;
243	unsigned long first;
244	unsigned long lastsrc;
245	unsigned long checksum = ~0U;
246
247	if (__get_word(ldq_u, first, src))
248		return 0;
249	lastsrc = 7+len+(unsigned long)src;
250	mskql(partial_dest, doff, partial_dest);
251	while (len >= 0) {
252		unsigned long second, word;
253		unsigned long second_dest;
254
255		if (__get_word(ldq_u, second, src+1))
256			return 0;
257		extql(first, soff, word);
258		checksum += carry;
259		len -= 8;
260		extqh(second, soff, first);
261		src++;
262		word |= first;
263		first = second;
264		insql(word, doff, second_dest);
265		checksum += word;
266		stq_u(partial_dest | second_dest, dst);
267		carry = checksum < word;
268		insqh(word, doff, partial_dest);
269		dst++;
270	}
271	len += doff;
272	checksum += carry;
273	if (len >= 0) {
274		unsigned long second, word;
275		unsigned long second_dest;
276
277		if (__get_word(ldq_u, second, lastsrc))
278			return 0;
279		extql(first, soff, word);
280		extqh(second, soff, first);
281		word |= first;
282		first = second;
283		mskql(word, len-doff, word);
284		checksum += word;
285		insql(word, doff, second_dest);
286		carry = checksum < word;
287		stq_u(partial_dest | second_dest, dst);
288		if (len) {
289			ldq_u(second_dest, dst+1);
290			insqh(word, doff, partial_dest);
291			mskqh(second_dest, len, second_dest);
292			stq_u(partial_dest | second_dest, dst+1);
293		}
294		checksum += carry;
295	} else {
296		unsigned long second, word;
297		unsigned long second_dest;
298
299		if (__get_word(ldq_u, second, lastsrc))
300			return 0;
301		extql(first, soff, word);
302		extqh(second, soff, first);
303		word |= first;
304		ldq_u(second_dest, dst);
305		mskql(word, len-doff, word);
306		checksum += word;
307		mskqh(second_dest, len, second_dest);
308		carry = checksum < word;
309		insql(word, doff, word);
310		stq_u(partial_dest | word | second_dest, dst);
311		checksum += carry;
312	}
313	return checksum;
314}
315
316static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
317{
318	unsigned long soff = 7 & (unsigned long) src;
319	unsigned long doff = 7 & (unsigned long) dst;
320	unsigned long checksum;
321
322	if (!doff) {
323		if (!soff)
324			checksum = csum_partial_cfu_aligned(
325				(const unsigned long __user *) src,
326				(unsigned long *) dst, len-8);
327		else
328			checksum = csum_partial_cfu_dest_aligned(
329				(const unsigned long __user *) src,
330				(unsigned long *) dst,
331				soff, len-8);
332	} else {
333		unsigned long partial_dest;
334		ldq_u(partial_dest, dst);
335		if (!soff)
336			checksum = csum_partial_cfu_src_aligned(
337				(const unsigned long __user *) src,
338				(unsigned long *) dst,
339				doff, len-8, partial_dest);
340		else
341			checksum = csum_partial_cfu_unaligned(
342				(const unsigned long __user *) src,
343				(unsigned long *) dst,
344				soff, doff, len-8, partial_dest);
345	}
346	return (__force __wsum)from64to16 (checksum);
347}
348
349__wsum
350csum_and_copy_from_user(const void __user *src, void *dst, int len)
351{
352	if (!access_ok(src, len))
353		return 0;
354	return __csum_and_copy(src, dst, len);
355}
356
357__wsum
358csum_partial_copy_nocheck(const void *src, void *dst, int len)
359{
360	return __csum_and_copy((__force const void __user *)src,
361						dst, len);
362}
363EXPORT_SYMBOL(csum_partial_copy_nocheck);
364