1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * This file contains kasan initialization code for ARM.
4 *
5 * Copyright (c) 2018 Samsung Electronics Co., Ltd.
6 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
7 * Author: Linus Walleij <linus.walleij@linaro.org>
8 */
9
10#define pr_fmt(fmt) "kasan: " fmt
11#include <linux/kasan.h>
12#include <linux/kernel.h>
13#include <linux/memblock.h>
14#include <linux/sched/task.h>
15#include <linux/start_kernel.h>
16#include <linux/pgtable.h>
17#include <asm/cputype.h>
18#include <asm/highmem.h>
19#include <asm/mach/map.h>
20#include <asm/page.h>
21#include <asm/pgalloc.h>
22#include <asm/procinfo.h>
23#include <asm/proc-fns.h>
24
25#include "mm.h"
26
27static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
28
29pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
30
31static __init void *kasan_alloc_block_raw(size_t size)
32{
33	return memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS),
34				      MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
35}
36
37static __init void *kasan_alloc_block(size_t size)
38{
39	return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
40				      MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
41}
42
43static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
44				      unsigned long end, bool early)
45{
46	unsigned long next;
47	pte_t *ptep = pte_offset_kernel(pmdp, addr);
48
49	do {
50		pte_t entry;
51		void *p;
52
53		next = addr + PAGE_SIZE;
54
55		if (!early) {
56			if (!pte_none(READ_ONCE(*ptep)))
57				continue;
58
59			p = kasan_alloc_block_raw(PAGE_SIZE);
60			if (!p) {
61				panic("%s failed to allocate shadow page for address 0x%lx\n",
62				      __func__, addr);
63				return;
64			}
65			memset(p, KASAN_SHADOW_INIT, PAGE_SIZE);
66			entry = pfn_pte(virt_to_pfn(p),
67					__pgprot(pgprot_val(PAGE_KERNEL)));
68		} else if (pte_none(READ_ONCE(*ptep))) {
69			/*
70			 * The early shadow memory is mapping all KASan
71			 * operations to one and the same page in memory,
72			 * "kasan_early_shadow_page" so that the instrumentation
73			 * will work on a scratch area until we can set up the
74			 * proper KASan shadow memory.
75			 */
76			entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page),
77					__pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN));
78		} else {
79			/*
80			 * Early shadow mappings are PMD_SIZE aligned, so if the
81			 * first entry is already set, they must all be set.
82			 */
83			return;
84		}
85
86		set_pte_at(&init_mm, addr, ptep, entry);
87	} while (ptep++, addr = next, addr != end);
88}
89
90/*
91 * The pmd (page middle directory) is only used on LPAE
92 */
93static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
94				      unsigned long end, bool early)
95{
96	unsigned long next;
97	pmd_t *pmdp = pmd_offset(pudp, addr);
98
99	do {
100		if (pmd_none(*pmdp)) {
101			/*
102			 * We attempt to allocate a shadow block for the PMDs
103			 * used by the PTEs for this address if it isn't already
104			 * allocated.
105			 */
106			void *p = early ? kasan_early_shadow_pte :
107				kasan_alloc_block(PAGE_SIZE);
108
109			if (!p) {
110				panic("%s failed to allocate shadow block for address 0x%lx\n",
111				      __func__, addr);
112				return;
113			}
114			pmd_populate_kernel(&init_mm, pmdp, p);
115			flush_pmd_entry(pmdp);
116		}
117
118		next = pmd_addr_end(addr, end);
119		kasan_pte_populate(pmdp, addr, next, early);
120	} while (pmdp++, addr = next, addr != end);
121}
122
123static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
124				      bool early)
125{
126	unsigned long next;
127	pgd_t *pgdp;
128	p4d_t *p4dp;
129	pud_t *pudp;
130
131	pgdp = pgd_offset_k(addr);
132
133	do {
134		/*
135		 * Allocate and populate the shadow block of p4d folded into
136		 * pud folded into pmd if it doesn't already exist
137		 */
138		if (!early && pgd_none(*pgdp)) {
139			void *p = kasan_alloc_block(PAGE_SIZE);
140
141			if (!p) {
142				panic("%s failed to allocate shadow block for address 0x%lx\n",
143				      __func__, addr);
144				return;
145			}
146			pgd_populate(&init_mm, pgdp, p);
147		}
148
149		next = pgd_addr_end(addr, end);
150		/*
151		 * We just immediately jump over the p4d and pud page
152		 * directories since we believe ARM32 will never gain four
153		 * nor five level page tables.
154		 */
155		p4dp = p4d_offset(pgdp, addr);
156		pudp = pud_offset(p4dp, addr);
157
158		kasan_pmd_populate(pudp, addr, next, early);
159	} while (pgdp++, addr = next, addr != end);
160}
161
162extern struct proc_info_list *lookup_processor_type(unsigned int);
163
164void __init kasan_early_init(void)
165{
166	struct proc_info_list *list;
167
168	/*
169	 * locate processor in the list of supported processor
170	 * types.  The linker builds this table for us from the
171	 * entries in arch/arm/mm/proc-*.S
172	 */
173	list = lookup_processor_type(read_cpuid_id());
174	if (list) {
175#ifdef MULTI_CPU
176		processor = *list->proc;
177#endif
178	}
179
180	BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
181	/*
182	 * We walk the page table and set all of the shadow memory to point
183	 * to the scratch page.
184	 */
185	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true);
186}
187
188static void __init clear_pgds(unsigned long start,
189			unsigned long end)
190{
191	for (; start && start < end; start += PMD_SIZE)
192		pmd_clear(pmd_off_k(start));
193}
194
195static int __init create_mapping(void *start, void *end)
196{
197	void *shadow_start, *shadow_end;
198
199	shadow_start = kasan_mem_to_shadow(start);
200	shadow_end = kasan_mem_to_shadow(end);
201
202	pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n",
203		start, end, shadow_start, shadow_end);
204
205	kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK,
206			   PAGE_ALIGN((unsigned long)shadow_end), false);
207	return 0;
208}
209
210void __init kasan_init(void)
211{
212	phys_addr_t pa_start, pa_end;
213	u64 i;
214
215	/*
216	 * We are going to perform proper setup of shadow memory.
217	 *
218	 * At first we should unmap early shadow (clear_pgds() call bellow).
219	 * However, instrumented code can't execute without shadow memory.
220	 *
221	 * To keep the early shadow memory MMU tables around while setting up
222	 * the proper shadow memory, we copy swapper_pg_dir (the initial page
223	 * table) to tmp_pgd_table and use that to keep the early shadow memory
224	 * mapped until the full shadow setup is finished. Then we swap back
225	 * to the proper swapper_pg_dir.
226	 */
227
228	memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
229#ifdef CONFIG_ARM_LPAE
230	/* We need to be in the same PGD or this won't work */
231	BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
232		     pgd_index(KASAN_SHADOW_END));
233	memcpy(tmp_pmd_table,
234	       (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
235	       sizeof(tmp_pmd_table));
236	set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
237		__pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
238#endif
239	cpu_switch_mm(tmp_pgd_table, &init_mm);
240	local_flush_tlb_all();
241
242	clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
243
244	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
245		kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
246					    kasan_mem_to_shadow((void *)VMALLOC_END));
247
248	kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_END),
249				    kasan_mem_to_shadow((void *)-1UL) + 1);
250
251	for_each_mem_range(i, &pa_start, &pa_end) {
252		void *start = __va(pa_start);
253		void *end = __va(pa_end);
254
255		/* Do not attempt to shadow highmem */
256		if (pa_start >= arm_lowmem_limit) {
257			pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end);
258			continue;
259		}
260		if (pa_end > arm_lowmem_limit) {
261			pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n",
262				&pa_start, &pa_end, &arm_lowmem_limit);
263			end = __va(arm_lowmem_limit);
264		}
265		if (start >= end) {
266			pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
267				&pa_start, &pa_end, start, end);
268			continue;
269		}
270
271		create_mapping(start, end);
272	}
273
274	/*
275	 * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
276	 *    so we need to map this area if CONFIG_KASAN_VMALLOC=n. With
277	 *    VMALLOC support KASAN will manage this region dynamically,
278	 *    refer to kasan_populate_vmalloc() and ARM's implementation of
279	 *    module_alloc().
280	 * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
281	 *    ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
282	 *    use kasan_populate_zero_shadow.
283	 */
284	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES))
285		create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END));
286	create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE));
287
288	/*
289	 * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
290	 * we should make sure that it maps the zero page read-only.
291	 */
292	for (i = 0; i < PTRS_PER_PTE; i++)
293		set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
294			   &kasan_early_shadow_pte[i],
295			   pfn_pte(virt_to_pfn(kasan_early_shadow_page),
296				__pgprot(pgprot_val(PAGE_KERNEL)
297					 | L_PTE_RDONLY)));
298
299	cpu_switch_mm(swapper_pg_dir, &init_mm);
300	local_flush_tlb_all();
301
302	memset(kasan_early_shadow_page, 0, PAGE_SIZE);
303	pr_info("Kernel address sanitizer initialized\n");
304	init_task.kasan_depth = 0;
305}
306