1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5#ifndef __ASM_MTE_KASAN_H
6#define __ASM_MTE_KASAN_H
7
8#include <asm/compiler.h>
9#include <asm/cputype.h>
10#include <asm/mte-def.h>
11
12#ifndef __ASSEMBLY__
13
14#include <linux/types.h>
15
16#ifdef CONFIG_KASAN_HW_TAGS
17
18/* Whether the MTE asynchronous mode is enabled. */
19DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
20
21static inline bool system_uses_mte_async_or_asymm_mode(void)
22{
23	return static_branch_unlikely(&mte_async_or_asymm_mode);
24}
25
26#else /* CONFIG_KASAN_HW_TAGS */
27
28static inline bool system_uses_mte_async_or_asymm_mode(void)
29{
30	return false;
31}
32
33#endif /* CONFIG_KASAN_HW_TAGS */
34
35#ifdef CONFIG_ARM64_MTE
36
37/*
38 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0
39 * affects EL0 and TCF affects EL1 irrespective of which TTBR is
40 * used.
41 * The kernel accesses TTBR0 usually with LDTR/STTR instructions
42 * when UAO is available, so these would act as EL0 accesses using
43 * TCF0.
44 * However futex.h code uses exclusives which would be executed as
45 * EL1, this can potentially cause a tag check fault even if the
46 * user disables TCF0.
47 *
48 * To address the problem we set the PSTATE.TCO bit in uaccess_enable()
49 * and reset it in uaccess_disable().
50 *
51 * The Tag check override (TCO) bit disables temporarily the tag checking
52 * preventing the issue.
53 */
54static inline void mte_disable_tco(void)
55{
56	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),
57				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
58}
59
60static inline void mte_enable_tco(void)
61{
62	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),
63				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
64}
65
66/*
67 * These functions disable tag checking only if in MTE async mode
68 * since the sync mode generates exceptions synchronously and the
69 * nofault or load_unaligned_zeropad can handle them.
70 */
71static inline void __mte_disable_tco_async(void)
72{
73	if (system_uses_mte_async_or_asymm_mode())
74		mte_disable_tco();
75}
76
77static inline void __mte_enable_tco_async(void)
78{
79	if (system_uses_mte_async_or_asymm_mode())
80		mte_enable_tco();
81}
82
83/*
84 * These functions are meant to be only used from KASAN runtime through
85 * the arch_*() interface defined in asm/memory.h.
86 * These functions don't include system_supports_mte() checks,
87 * as KASAN only calls them when MTE is supported and enabled.
88 */
89
90static inline u8 mte_get_ptr_tag(void *ptr)
91{
92	/* Note: The format of KASAN tags is 0xF<x> */
93	u8 tag = 0xF0 | (u8)(((u64)(ptr)) >> MTE_TAG_SHIFT);
94
95	return tag;
96}
97
98/* Get allocation tag for the address. */
99static inline u8 mte_get_mem_tag(void *addr)
100{
101	asm(__MTE_PREAMBLE "ldg %0, [%0]"
102		: "+r" (addr));
103
104	return mte_get_ptr_tag(addr);
105}
106
107/* Generate a random tag. */
108static inline u8 mte_get_random_tag(void)
109{
110	void *addr;
111
112	asm(__MTE_PREAMBLE "irg %0, %0"
113		: "=r" (addr));
114
115	return mte_get_ptr_tag(addr);
116}
117
118static inline u64 __stg_post(u64 p)
119{
120	asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16"
121		     : "+r"(p)
122		     :
123		     : "memory");
124	return p;
125}
126
127static inline u64 __stzg_post(u64 p)
128{
129	asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16"
130		     : "+r"(p)
131		     :
132		     : "memory");
133	return p;
134}
135
136static inline void __dc_gva(u64 p)
137{
138	asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory");
139}
140
141static inline void __dc_gzva(u64 p)
142{
143	asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory");
144}
145
146/*
147 * Assign allocation tags for a region of memory based on the pointer tag.
148 * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
149 * size must be MTE_GRANULE_SIZE aligned.
150 */
151static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
152					 bool init)
153{
154	u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3;
155
156	/* Read DC G(Z)VA block size from the system register. */
157	dczid = read_cpuid(DCZID_EL0);
158	dczid_bs = 4ul << (dczid & 0xf);
159	dczid_dzp = (dczid >> 4) & 1;
160
161	curr = (u64)__tag_set(addr, tag);
162	mask = dczid_bs - 1;
163	/* STG/STZG up to the end of the first block. */
164	end1 = curr | mask;
165	end3 = curr + size;
166	/* DC GVA / GZVA in [end1, end2) */
167	end2 = end3 & ~mask;
168
169	/*
170	 * The following code uses STG on the first DC GVA block even if the
171	 * start address is aligned - it appears to be faster than an alignment
172	 * check + conditional branch. Also, if the range size is at least 2 DC
173	 * GVA blocks, the first two loops can use post-condition to save one
174	 * branch each.
175	 */
176#define SET_MEMTAG_RANGE(stg_post, dc_gva)		\
177	do {						\
178		if (!dczid_dzp && size >= 2 * dczid_bs) {\
179			do {				\
180				curr = stg_post(curr);	\
181			} while (curr < end1);		\
182							\
183			do {				\
184				dc_gva(curr);		\
185				curr += dczid_bs;	\
186			} while (curr < end2);		\
187		}					\
188							\
189		while (curr < end3)			\
190			curr = stg_post(curr);		\
191	} while (0)
192
193	if (init)
194		SET_MEMTAG_RANGE(__stzg_post, __dc_gzva);
195	else
196		SET_MEMTAG_RANGE(__stg_post, __dc_gva);
197#undef SET_MEMTAG_RANGE
198}
199
200void mte_enable_kernel_sync(void);
201void mte_enable_kernel_async(void);
202void mte_enable_kernel_asymm(void);
203
204#else /* CONFIG_ARM64_MTE */
205
206static inline void mte_disable_tco(void)
207{
208}
209
210static inline void mte_enable_tco(void)
211{
212}
213
214static inline void __mte_disable_tco_async(void)
215{
216}
217
218static inline void __mte_enable_tco_async(void)
219{
220}
221
222static inline u8 mte_get_ptr_tag(void *ptr)
223{
224	return 0xFF;
225}
226
227static inline u8 mte_get_mem_tag(void *addr)
228{
229	return 0xFF;
230}
231
232static inline u8 mte_get_random_tag(void)
233{
234	return 0xFF;
235}
236
237static inline void mte_set_mem_tag_range(void *addr, size_t size,
238						u8 tag, bool init)
239{
240}
241
242static inline void mte_enable_kernel_sync(void)
243{
244}
245
246static inline void mte_enable_kernel_async(void)
247{
248}
249
250static inline void mte_enable_kernel_asymm(void)
251{
252}
253
254#endif /* CONFIG_ARM64_MTE */
255
256#endif /* __ASSEMBLY__ */
257
258#endif /* __ASM_MTE_KASAN_H  */
259