1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Accelerated CRC32(C) using AArch64 CRC instructions
4 *
5 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8#include <linux/linkage.h>
9#include <asm/alternative.h>
10#include <asm/assembler.h>
11
12	.arch		armv8-a+crc
13
14	.macro		byteorder, reg, be
15	.if		\be
16CPU_LE( rev		\reg, \reg	)
17	.else
18CPU_BE( rev		\reg, \reg	)
19	.endif
20	.endm
21
22	.macro		byteorder16, reg, be
23	.if		\be
24CPU_LE( rev16		\reg, \reg	)
25	.else
26CPU_BE( rev16		\reg, \reg	)
27	.endif
28	.endm
29
30	.macro		bitorder, reg, be
31	.if		\be
32	rbit		\reg, \reg
33	.endif
34	.endm
35
36	.macro		bitorder16, reg, be
37	.if		\be
38	rbit		\reg, \reg
39	lsr		\reg, \reg, #16
40	.endif
41	.endm
42
43	.macro		bitorder8, reg, be
44	.if		\be
45	rbit		\reg, \reg
46	lsr		\reg, \reg, #24
47	.endif
48	.endm
49
50	.macro		__crc32, c, be=0
51	bitorder	w0, \be
52	cmp		x2, #16
53	b.lt		8f			// less than 16 bytes
54
55	and		x7, x2, #0x1f
56	and		x2, x2, #~0x1f
57	cbz		x7, 32f			// multiple of 32 bytes
58
59	and		x8, x7, #0xf
60	ldp		x3, x4, [x1]
61	add		x8, x8, x1
62	add		x1, x1, x7
63	ldp		x5, x6, [x8]
64	byteorder	x3, \be
65	byteorder	x4, \be
66	byteorder	x5, \be
67	byteorder	x6, \be
68	bitorder	x3, \be
69	bitorder	x4, \be
70	bitorder	x5, \be
71	bitorder	x6, \be
72
73	tst		x7, #8
74	crc32\c\()x	w8, w0, x3
75	csel		x3, x3, x4, eq
76	csel		w0, w0, w8, eq
77	tst		x7, #4
78	lsr		x4, x3, #32
79	crc32\c\()w	w8, w0, w3
80	csel		x3, x3, x4, eq
81	csel		w0, w0, w8, eq
82	tst		x7, #2
83	lsr		w4, w3, #16
84	crc32\c\()h	w8, w0, w3
85	csel		w3, w3, w4, eq
86	csel		w0, w0, w8, eq
87	tst		x7, #1
88	crc32\c\()b	w8, w0, w3
89	csel		w0, w0, w8, eq
90	tst		x7, #16
91	crc32\c\()x	w8, w0, x5
92	crc32\c\()x	w8, w8, x6
93	csel		w0, w0, w8, eq
94	cbz		x2, 0f
95
9632:	ldp		x3, x4, [x1], #32
97	sub		x2, x2, #32
98	ldp		x5, x6, [x1, #-16]
99	byteorder	x3, \be
100	byteorder	x4, \be
101	byteorder	x5, \be
102	byteorder	x6, \be
103	bitorder	x3, \be
104	bitorder	x4, \be
105	bitorder	x5, \be
106	bitorder	x6, \be
107	crc32\c\()x	w0, w0, x3
108	crc32\c\()x	w0, w0, x4
109	crc32\c\()x	w0, w0, x5
110	crc32\c\()x	w0, w0, x6
111	cbnz		x2, 32b
1120:	bitorder	w0, \be
113	ret
114
1158:	tbz		x2, #3, 4f
116	ldr		x3, [x1], #8
117	byteorder	x3, \be
118	bitorder	x3, \be
119	crc32\c\()x	w0, w0, x3
1204:	tbz		x2, #2, 2f
121	ldr		w3, [x1], #4
122	byteorder	w3, \be
123	bitorder	w3, \be
124	crc32\c\()w	w0, w0, w3
1252:	tbz		x2, #1, 1f
126	ldrh		w3, [x1], #2
127	byteorder16	w3, \be
128	bitorder16	w3, \be
129	crc32\c\()h	w0, w0, w3
1301:	tbz		x2, #0, 0f
131	ldrb		w3, [x1]
132	bitorder8	w3, \be
133	crc32\c\()b	w0, w0, w3
1340:	bitorder	w0, \be
135	ret
136	.endm
137
138	.align		5
139SYM_FUNC_START(crc32_le)
140alternative_if_not ARM64_HAS_CRC32
141	b		crc32_le_base
142alternative_else_nop_endif
143	__crc32
144SYM_FUNC_END(crc32_le)
145
146	.align		5
147SYM_FUNC_START(__crc32c_le)
148alternative_if_not ARM64_HAS_CRC32
149	b		__crc32c_le_base
150alternative_else_nop_endif
151	__crc32		c
152SYM_FUNC_END(__crc32c_le)
153
154	.align		5
155SYM_FUNC_START(crc32_be)
156alternative_if_not ARM64_HAS_CRC32
157	b		crc32_be_base
158alternative_else_nop_endif
159	__crc32		be=1
160SYM_FUNC_END(crc32_be)
161