1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5#ifndef __ASM_MTE_H
6#define __ASM_MTE_H
7
8#include <asm/compiler.h>
9#include <asm/mte-def.h>
10
11#ifndef __ASSEMBLY__
12
13#include <linux/bitfield.h>
14#include <linux/kasan-enabled.h>
15#include <linux/page-flags.h>
16#include <linux/sched.h>
17#include <linux/types.h>
18
19#include <asm/pgtable-types.h>
20
21void mte_clear_page_tags(void *addr);
22unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
23				      unsigned long n);
24unsigned long mte_copy_tags_to_user(void __user *to, void *from,
25				    unsigned long n);
26int mte_save_tags(struct page *page);
27void mte_save_page_tags(const void *page_addr, void *tag_storage);
28void mte_restore_tags(swp_entry_t entry, struct page *page);
29void mte_restore_page_tags(void *page_addr, const void *tag_storage);
30void mte_invalidate_tags(int type, pgoff_t offset);
31void mte_invalidate_tags_area(int type);
32void *mte_allocate_tag_storage(void);
33void mte_free_tag_storage(char *storage);
34
35#ifdef CONFIG_ARM64_MTE
36
37/* track which pages have valid allocation tags */
38#define PG_mte_tagged	PG_arch_2
39/* simple lock to avoid multiple threads tagging the same page */
40#define PG_mte_lock	PG_arch_3
41
42static inline void set_page_mte_tagged(struct page *page)
43{
44	/*
45	 * Ensure that the tags written prior to this function are visible
46	 * before the page flags update.
47	 */
48	smp_wmb();
49	set_bit(PG_mte_tagged, &page->flags);
50}
51
52static inline bool page_mte_tagged(struct page *page)
53{
54	bool ret = test_bit(PG_mte_tagged, &page->flags);
55
56	/*
57	 * If the page is tagged, ensure ordering with a likely subsequent
58	 * read of the tags.
59	 */
60	if (ret)
61		smp_rmb();
62	return ret;
63}
64
65/*
66 * Lock the page for tagging and return 'true' if the page can be tagged,
67 * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the
68 * locking only happens once for page initialisation.
69 *
70 * The page MTE lock state:
71 *
72 *   Locked:	PG_mte_lock && !PG_mte_tagged
73 *   Unlocked:	!PG_mte_lock || PG_mte_tagged
74 *
75 * Acquire semantics only if the page is tagged (returning 'false').
76 */
77static inline bool try_page_mte_tagging(struct page *page)
78{
79	if (!test_and_set_bit(PG_mte_lock, &page->flags))
80		return true;
81
82	/*
83	 * The tags are either being initialised or may have been initialised
84	 * already. Check if the PG_mte_tagged flag has been set or wait
85	 * otherwise.
86	 */
87	smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged));
88
89	return false;
90}
91
92void mte_zero_clear_page_tags(void *addr);
93void mte_sync_tags(pte_t pte, unsigned int nr_pages);
94void mte_copy_page_tags(void *kto, const void *kfrom);
95void mte_thread_init_user(void);
96void mte_thread_switch(struct task_struct *next);
97void mte_cpu_setup(void);
98void mte_suspend_enter(void);
99void mte_suspend_exit(void);
100long set_mte_ctrl(struct task_struct *task, unsigned long arg);
101long get_mte_ctrl(struct task_struct *task);
102int mte_ptrace_copy_tags(struct task_struct *child, long request,
103			 unsigned long addr, unsigned long data);
104size_t mte_probe_user_range(const char __user *uaddr, size_t size);
105
106#else /* CONFIG_ARM64_MTE */
107
108/* unused if !CONFIG_ARM64_MTE, silence the compiler */
109#define PG_mte_tagged	0
110
111static inline void set_page_mte_tagged(struct page *page)
112{
113}
114static inline bool page_mte_tagged(struct page *page)
115{
116	return false;
117}
118static inline bool try_page_mte_tagging(struct page *page)
119{
120	return false;
121}
122static inline void mte_zero_clear_page_tags(void *addr)
123{
124}
125static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages)
126{
127}
128static inline void mte_copy_page_tags(void *kto, const void *kfrom)
129{
130}
131static inline void mte_thread_init_user(void)
132{
133}
134static inline void mte_thread_switch(struct task_struct *next)
135{
136}
137static inline void mte_suspend_enter(void)
138{
139}
140static inline void mte_suspend_exit(void)
141{
142}
143static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
144{
145	return 0;
146}
147static inline long get_mte_ctrl(struct task_struct *task)
148{
149	return 0;
150}
151static inline int mte_ptrace_copy_tags(struct task_struct *child,
152				       long request, unsigned long addr,
153				       unsigned long data)
154{
155	return -EIO;
156}
157
158#endif /* CONFIG_ARM64_MTE */
159
160static inline void mte_disable_tco_entry(struct task_struct *task)
161{
162	if (!system_supports_mte())
163		return;
164
165	/*
166	 * Re-enable tag checking (TCO set on exception entry). This is only
167	 * necessary if MTE is enabled in either the kernel or the userspace
168	 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set
169	 * for both). With MTE disabled in the kernel and disabled or
170	 * asynchronous in userspace, tag check faults (including in uaccesses)
171	 * are not reported, therefore there is no need to re-enable checking.
172	 * This is beneficial on microarchitectures where re-enabling TCO is
173	 * expensive.
174	 */
175	if (kasan_hw_tags_enabled() ||
176	    (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))
177		asm volatile(SET_PSTATE_TCO(0));
178}
179
180#ifdef CONFIG_KASAN_HW_TAGS
181void mte_check_tfsr_el1(void);
182
183static inline void mte_check_tfsr_entry(void)
184{
185	if (!system_supports_mte())
186		return;
187
188	mte_check_tfsr_el1();
189}
190
191static inline void mte_check_tfsr_exit(void)
192{
193	if (!system_supports_mte())
194		return;
195
196	/*
197	 * The asynchronous faults are sync'ed automatically with
198	 * TFSR_EL1 on kernel entry but for exit an explicit dsb()
199	 * is required.
200	 */
201	dsb(nsh);
202	isb();
203
204	mte_check_tfsr_el1();
205}
206#else
207static inline void mte_check_tfsr_el1(void)
208{
209}
210static inline void mte_check_tfsr_entry(void)
211{
212}
213static inline void mte_check_tfsr_exit(void)
214{
215}
216#endif /* CONFIG_KASAN_HW_TAGS */
217
218#endif /* __ASSEMBLY__ */
219#endif /* __ASM_MTE_H  */
220