1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_ALTERNATIVE_MACROS_H
3#define __ASM_ALTERNATIVE_MACROS_H
4
5#include <linux/const.h>
6#include <vdso/bits.h>
7
8#include <asm/cpucaps.h>
9#include <asm/insn-def.h>
10
11/*
12 * Binutils 2.27.0 can't handle a 'UL' suffix on constants, so for the assembly
13 * macros below we must use we must use `(1 << ARM64_CB_SHIFT)`.
14 */
15#define ARM64_CB_SHIFT	15
16#define ARM64_CB_BIT	BIT(ARM64_CB_SHIFT)
17
18#if ARM64_NCAPS >= ARM64_CB_BIT
19#error "cpucaps have overflown ARM64_CB_BIT"
20#endif
21
22#ifndef __ASSEMBLY__
23
24#include <linux/stringify.h>
25
26#define ALTINSTR_ENTRY(cpucap)					              \
27	" .word 661b - .\n"				/* label           */ \
28	" .word 663f - .\n"				/* new instruction */ \
29	" .hword " __stringify(cpucap) "\n"		/* cpucap          */ \
30	" .byte 662b-661b\n"				/* source len      */ \
31	" .byte 664f-663f\n"				/* replacement len */
32
33#define ALTINSTR_ENTRY_CB(cpucap, cb)					      \
34	" .word 661b - .\n"				/* label           */ \
35	" .word " __stringify(cb) "- .\n"		/* callback        */ \
36	" .hword " __stringify(cpucap) "\n"		/* cpucap          */ \
37	" .byte 662b-661b\n"				/* source len      */ \
38	" .byte 664f-663f\n"				/* replacement len */
39
40/*
41 * alternative assembly primitive:
42 *
43 * If any of these .org directive fail, it means that insn1 and insn2
44 * don't have the same length. This used to be written as
45 *
46 * .if ((664b-663b) != (662b-661b))
47 * 	.error "Alternatives instruction length mismatch"
48 * .endif
49 *
50 * but most assemblers die if insn1 or insn2 have a .inst. This should
51 * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
52 * containing commit 4e4d08cf7399b606 or c1baaddf8861).
53 *
54 * Alternatives with callbacks do not generate replacement instructions.
55 */
56#define __ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg_enabled)	\
57	".if "__stringify(cfg_enabled)" == 1\n"				\
58	"661:\n\t"							\
59	oldinstr "\n"							\
60	"662:\n"							\
61	".pushsection .altinstructions,\"a\"\n"				\
62	ALTINSTR_ENTRY(cpucap)						\
63	".popsection\n"							\
64	".subsection 1\n"						\
65	"663:\n\t"							\
66	newinstr "\n"							\
67	"664:\n\t"							\
68	".org	. - (664b-663b) + (662b-661b)\n\t"			\
69	".org	. - (662b-661b) + (664b-663b)\n\t"			\
70	".previous\n"							\
71	".endif\n"
72
73#define __ALTERNATIVE_CFG_CB(oldinstr, cpucap, cfg_enabled, cb)	\
74	".if "__stringify(cfg_enabled)" == 1\n"				\
75	"661:\n\t"							\
76	oldinstr "\n"							\
77	"662:\n"							\
78	".pushsection .altinstructions,\"a\"\n"				\
79	ALTINSTR_ENTRY_CB(cpucap, cb)					\
80	".popsection\n"							\
81	"663:\n\t"							\
82	"664:\n\t"							\
83	".endif\n"
84
85#define _ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg, ...)	\
86	__ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, IS_ENABLED(cfg))
87
88#define ALTERNATIVE_CB(oldinstr, cpucap, cb) \
89	__ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (cpucap), 1, cb)
90#else
91
92#include <asm/assembler.h>
93
94.macro altinstruction_entry orig_offset alt_offset cpucap orig_len alt_len
95	.word \orig_offset - .
96	.word \alt_offset - .
97	.hword (\cpucap)
98	.byte \orig_len
99	.byte \alt_len
100.endm
101
102.macro alternative_insn insn1, insn2, cap, enable = 1
103	.if \enable
104661:	\insn1
105662:	.pushsection .altinstructions, "a"
106	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
107	.popsection
108	.subsection 1
109663:	\insn2
110664:	.org	. - (664b-663b) + (662b-661b)
111	.org	. - (662b-661b) + (664b-663b)
112	.previous
113	.endif
114.endm
115
116/*
117 * Alternative sequences
118 *
119 * The code for the case where the capability is not present will be
120 * assembled and linked as normal. There are no restrictions on this
121 * code.
122 *
123 * The code for the case where the capability is present will be
124 * assembled into a special section to be used for dynamic patching.
125 * Code for that case must:
126 *
127 * 1. Be exactly the same length (in bytes) as the default code
128 *    sequence.
129 *
130 * 2. Not contain a branch target that is used outside of the
131 *    alternative sequence it is defined in (branches into an
132 *    alternative sequence are not fixed up).
133 */
134
135/*
136 * Begin an alternative code sequence.
137 */
138.macro alternative_if_not cap
139	.set .Lasm_alt_mode, 0
140	.pushsection .altinstructions, "a"
141	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
142	.popsection
143661:
144.endm
145
146.macro alternative_if cap
147	.set .Lasm_alt_mode, 1
148	.pushsection .altinstructions, "a"
149	altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
150	.popsection
151	.subsection 1
152	.align 2	/* So GAS knows label 661 is suitably aligned */
153661:
154.endm
155
156.macro alternative_cb cap, cb
157	.set .Lasm_alt_mode, 0
158	.pushsection .altinstructions, "a"
159	altinstruction_entry 661f, \cb, (1 << ARM64_CB_SHIFT) | \cap, 662f-661f, 0
160	.popsection
161661:
162.endm
163
164/*
165 * Provide the other half of the alternative code sequence.
166 */
167.macro alternative_else
168662:
169	.if .Lasm_alt_mode==0
170	.subsection 1
171	.else
172	.previous
173	.endif
174663:
175.endm
176
177/*
178 * Complete an alternative code sequence.
179 */
180.macro alternative_endif
181664:
182	.org	. - (664b-663b) + (662b-661b)
183	.org	. - (662b-661b) + (664b-663b)
184	.if .Lasm_alt_mode==0
185	.previous
186	.endif
187.endm
188
189/*
190 * Callback-based alternative epilogue
191 */
192.macro alternative_cb_end
193662:
194.endm
195
196/*
197 * Provides a trivial alternative or default sequence consisting solely
198 * of NOPs. The number of NOPs is chosen automatically to match the
199 * previous case.
200 */
201.macro alternative_else_nop_endif
202alternative_else
203	nops	(662b-661b) / AARCH64_INSN_SIZE
204alternative_endif
205.endm
206
207#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
208	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
209
210#endif  /*  __ASSEMBLY__  */
211
212/*
213 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap));
214 *
215 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap, CONFIG_FOO));
216 * N.B. If CONFIG_FOO is specified, but not selected, the whole block
217 *      will be omitted, including oldinstr.
218 */
219#define ALTERNATIVE(oldinstr, newinstr, ...)   \
220	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
221
222#ifndef __ASSEMBLY__
223
224#include <linux/types.h>
225
226static __always_inline bool
227alternative_has_cap_likely(const unsigned long cpucap)
228{
229	if (!cpucap_is_possible(cpucap))
230		return false;
231
232	asm goto(
233	ALTERNATIVE_CB("b	%l[l_no]", %[cpucap], alt_cb_patch_nops)
234	:
235	: [cpucap] "i" (cpucap)
236	:
237	: l_no);
238
239	return true;
240l_no:
241	return false;
242}
243
244static __always_inline bool
245alternative_has_cap_unlikely(const unsigned long cpucap)
246{
247	if (!cpucap_is_possible(cpucap))
248		return false;
249
250	asm goto(
251	ALTERNATIVE("nop", "b	%l[l_yes]", %[cpucap])
252	:
253	: [cpucap] "i" (cpucap)
254	:
255	: l_yes);
256
257	return false;
258l_yes:
259	return true;
260}
261
262#endif /* __ASSEMBLY__ */
263
264#endif /* __ASM_ALTERNATIVE_MACROS_H */
265