mm/kasan/init.c

92 static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr,
95 	pte_t *pte = pte_offset_kernel(pmd, addr);
102 	while (addr + PAGE_SIZE <= end) {
103 		set_pte_at(&init_mm, addr, pte, zero_pte);
104 		addr += PAGE_SIZE;
105 		pte = pte_offset_kernel(pmd, addr);
109 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
112 	pmd_t *pmd = pmd_offset(pud, addr);
116 		next = pmd_addr_end(addr, end);
118 		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
136 		zero_pte_populate(pmd, addr, next);
137 	} while (pmd++, addr = next, addr != end);
142 void __weak __meminit pmd_init(void *addr)
146 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr,
149 	pud_t *pud = pud_offset(p4d, addr);
153 		next = pud_addr_end(addr, end);
154 		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
159 			pmd = pmd_offset(pud, addr);
169 				p = pmd_alloc(&init_mm, pud, addr);
178 		zero_pmd_populate(pud, addr, next);
179 	} while (pud++, addr = next, addr != end);
184 void __weak __meminit pud_init(void *addr)
188 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
191 	p4d_t *p4d = p4d_offset(pgd, addr);
195 		next = p4d_addr_end(addr, end);
196 		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
202 			pud = pud_offset(p4d, addr);
205 			pmd = pmd_offset(pud, addr);
215 				p = pud_alloc(&init_mm, p4d, addr);
224 		zero_pud_populate(p4d, addr, next);
225 	} while (p4d++, addr = next, addr != end);
239 	unsigned long addr = (unsigned long)shadow_start;
241 	pgd_t *pgd = pgd_offset_k(addr);
245 		next = pgd_addr_end(addr, end);
247 		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
262 			p4d = p4d_offset(pgd, addr);
265 			pud = pud_offset(p4d, addr);
268 			pmd = pmd_offset(pud, addr);
278 				p = p4d_alloc(&init_mm, pgd, addr);
286 		zero_p4d_populate(pgd, addr, next);
287 	} while (pgd++, addr = next, addr != end);
352 static void kasan_remove_pte_table(pte_t *pte, unsigned long addr,
358 	for (; addr < end; addr = next, pte++) {
359 		next = (addr + PAGE_SIZE) & PAGE_MASK;
370 		pte_clear(&init_mm, addr, pte);
374 static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,
379 	for (; addr < end; addr = next, pmd++) {
382 		next = pmd_addr_end(addr, end);
388 			if (IS_ALIGNED(addr, PMD_SIZE) &&
394 		pte = pte_offset_kernel(pmd, addr);
395 		kasan_remove_pte_table(pte, addr, next);
400 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,
405 	for (; addr < end; addr = next, pud++) {
408 		next = pud_addr_end(addr, end);
414 			if (IS_ALIGNED(addr, PUD_SIZE) &&
420 		pmd = pmd_offset(pud, addr);
422 		kasan_remove_pmd_table(pmd, addr, next);
427 static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr,
432 	for (; addr < end; addr = next, p4d++) {
435 		next = p4d_addr_end(addr, end);
441 			if (IS_ALIGNED(addr, P4D_SIZE) &&
447 		pud = pud_offset(p4d, addr);
448 		kasan_remove_pud_table(pud, addr, next);
455 	unsigned long addr, end, next;
458 	addr = (unsigned long)kasan_mem_to_shadow(start);
459 	end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT);
465 	for (; addr < end; addr = next) {
468 		next = pgd_addr_end(addr, end);
470 		pgd = pgd_offset_k(addr);
475 			if (IS_ALIGNED(addr, PGDIR_SIZE) &&
482 		p4d = p4d_offset(pgd, addr);
483 		kasan_remove_p4d_table(p4d, addr, next);