1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2012-2022, Arm Limited.
4 *
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S
7 */
8
9#include <linux/linkage.h>
10#include <asm/assembler.h>
11
12/* Assumptions:
13 *
14 * ARMv8-a, AArch64.
15 * MTE compatible.
16 */
17
18#define L(label) .L ## label
19
20#define REP8_01 0x0101010101010101
21#define REP8_7f 0x7f7f7f7f7f7f7f7f
22
23#define src1		x0
24#define src2		x1
25#define result		x0
26
27#define data1		x2
28#define data1w		w2
29#define data2		x3
30#define data2w		w3
31#define has_nul		x4
32#define diff		x5
33#define off1		x5
34#define syndrome	x6
35#define tmp		x6
36#define data3		x7
37#define zeroones	x8
38#define shift		x9
39#define off2		x10
40
41/* On big-endian early bytes are at MSB and on little-endian LSB.
42   LS_FW means shifting towards early bytes.  */
43#ifdef __AARCH64EB__
44# define LS_FW lsl
45#else
46# define LS_FW lsr
47#endif
48
49/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
50   (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
51   can be done in parallel across the entire word.
52   Since carry propagation makes 0x1 bytes before a NUL byte appear
53   NUL too in big-endian, byte-reverse the data before the NUL check.  */
54
55
56SYM_FUNC_START(__pi_strcmp)
57	sub	off2, src2, src1
58	mov	zeroones, REP8_01
59	and	tmp, src1, 7
60	tst	off2, 7
61	b.ne	L(misaligned8)
62	cbnz	tmp, L(mutual_align)
63
64	.p2align 4
65
66L(loop_aligned):
67	ldr	data2, [src1, off2]
68	ldr	data1, [src1], 8
69L(start_realigned):
70#ifdef __AARCH64EB__
71	rev	tmp, data1
72	sub	has_nul, tmp, zeroones
73	orr	tmp, tmp, REP8_7f
74#else
75	sub	has_nul, data1, zeroones
76	orr	tmp, data1, REP8_7f
77#endif
78	bics	has_nul, has_nul, tmp	/* Non-zero if NUL terminator.  */
79	ccmp	data1, data2, 0, eq
80	b.eq	L(loop_aligned)
81#ifdef __AARCH64EB__
82	rev	has_nul, has_nul
83#endif
84	eor	diff, data1, data2
85	orr	syndrome, diff, has_nul
86L(end):
87#ifndef __AARCH64EB__
88	rev	syndrome, syndrome
89	rev	data1, data1
90	rev	data2, data2
91#endif
92	clz	shift, syndrome
93	/* The most-significant-non-zero bit of the syndrome marks either the
94	   first bit that is different, or the top bit of the first zero byte.
95	   Shifting left now will bring the critical information into the
96	   top bits.  */
97	lsl	data1, data1, shift
98	lsl	data2, data2, shift
99	/* But we need to zero-extend (char is unsigned) the value and then
100	   perform a signed 32-bit subtraction.  */
101	lsr	data1, data1, 56
102	sub	result, data1, data2, lsr 56
103	ret
104
105	.p2align 4
106
107L(mutual_align):
108	/* Sources are mutually aligned, but are not currently at an
109	   alignment boundary.  Round down the addresses and then mask off
110	   the bytes that precede the start point.  */
111	bic	src1, src1, 7
112	ldr	data2, [src1, off2]
113	ldr	data1, [src1], 8
114	neg	shift, src2, lsl 3	/* Bits to alignment -64.  */
115	mov	tmp, -1
116	LS_FW	tmp, tmp, shift
117	orr	data1, data1, tmp
118	orr	data2, data2, tmp
119	b	L(start_realigned)
120
121L(misaligned8):
122	/* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
123	   checking to make sure that we don't access beyond the end of SRC2.  */
124	cbz	tmp, L(src1_aligned)
125L(do_misaligned):
126	ldrb	data1w, [src1], 1
127	ldrb	data2w, [src2], 1
128	cmp	data1w, 0
129	ccmp	data1w, data2w, 0, ne	/* NZCV = 0b0000.  */
130	b.ne	L(done)
131	tst	src1, 7
132	b.ne	L(do_misaligned)
133
134L(src1_aligned):
135	neg	shift, src2, lsl 3
136	bic	src2, src2, 7
137	ldr	data3, [src2], 8
138#ifdef __AARCH64EB__
139	rev	data3, data3
140#endif
141	lsr	tmp, zeroones, shift
142	orr	data3, data3, tmp
143	sub	has_nul, data3, zeroones
144	orr	tmp, data3, REP8_7f
145	bics	has_nul, has_nul, tmp
146	b.ne	L(tail)
147
148	sub	off1, src2, src1
149
150	.p2align 4
151
152L(loop_unaligned):
153	ldr	data3, [src1, off1]
154	ldr	data2, [src1, off2]
155#ifdef __AARCH64EB__
156	rev	data3, data3
157#endif
158	sub	has_nul, data3, zeroones
159	orr	tmp, data3, REP8_7f
160	ldr	data1, [src1], 8
161	bics	has_nul, has_nul, tmp
162	ccmp	data1, data2, 0, eq
163	b.eq	L(loop_unaligned)
164
165	lsl	tmp, has_nul, shift
166#ifdef __AARCH64EB__
167	rev	tmp, tmp
168#endif
169	eor	diff, data1, data2
170	orr	syndrome, diff, tmp
171	cbnz	syndrome, L(end)
172L(tail):
173	ldr	data1, [src1]
174	neg	shift, shift
175	lsr	data2, data3, shift
176	lsr	has_nul, has_nul, shift
177#ifdef __AARCH64EB__
178	rev     data2, data2
179	rev	has_nul, has_nul
180#endif
181	eor	diff, data1, data2
182	orr	syndrome, diff, has_nul
183	b	L(end)
184
185L(done):
186	sub	result, data1, data2
187	ret
188SYM_FUNC_END(__pi_strcmp)
189SYM_FUNC_ALIAS_WEAK(strcmp, __pi_strcmp)
190EXPORT_SYMBOL_NOKASAN(strcmp)
191