linux-master/mm/debug_vm_pgtable.c

92 static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx)
95 	pte_t pte = pfn_pte(args->fixed_pte_pfn, prot);
112 	WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte), args->vma)));
115 	WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte, args->vma))));
120 static void __init pte_advanced_tests(struct pgtable_debug_args *args)
136 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;
141 	if (WARN_ON(!args->ptep))
144 	pte = pfn_pte(args->pte_pfn, args->page_prot);
145 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);
147 	ptep_set_wrprotect(args->mm, args->vaddr, args->ptep);
148 	pte = ptep_get(args->ptep);
150 	ptep_get_and_clear(args->mm, args->vaddr, args->ptep);
151 	pte = ptep_get(args->ptep);
154 	pte = pfn_pte(args->pte_pfn, args->page_prot);
157 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);
159 	pte = pte_mkwrite(pte, args->vma);
161 	ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);
162 	pte = ptep_get(args->ptep);
164 	ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);
165 	pte = ptep_get(args->ptep);
168 	pte = pfn_pte(args->pte_pfn, args->page_prot);
170 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);
172 	ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep);
173 	pte = ptep_get(args->ptep);
176 	ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);
180 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx)
190 	pmd = pfn_pmd(args->fixed_pmd_pfn, prot);
205 	WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd), args->vma)));
208 	WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd, args->vma))));
218 static void __init pmd_advanced_tests(struct pgtable_debug_args *args)
222 	unsigned long vaddr = args->vaddr;
227 	page = (args->pmd_pfn != ULONG_MAX) ? pfn_to_page(args->pmd_pfn) : NULL;
242 	pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep);
244 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);
245 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);
247 	pmdp_set_wrprotect(args->mm, vaddr, args->pmdp);
248 	pmd = READ_ONCE(*args->pmdp);
250 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);
251 	pmd = READ_ONCE(*args->pmdp);
254 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);
257 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);
259 	pmd = pmd_mkwrite(pmd, args->vma);
261 	pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);
262 	pmd = READ_ONCE(*args->pmdp);
264 	pmdp_huge_get_and_clear_full(args->vma, vaddr, args->pmdp, 1);
265 	pmd = READ_ONCE(*args->pmdp);
268 	pmd = pmd_mkhuge(pfn_pmd(args->pmd_pfn, args->page_prot));
270 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);
272 	pmdp_test_and_clear_young(args->vma, vaddr, args->pmdp);
273 	pmd = READ_ONCE(*args->pmdp);
277 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);
278 	pgtable_trans_huge_withdraw(args->mm, args->pmdp);
281 static void __init pmd_leaf_tests(struct pgtable_debug_args *args)
289 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
299 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx)
309 	pud = pfn_pud(args->fixed_pud_pfn, prot);
330 	if (mm_pmd_folded(args->mm))
340 static void __init pud_advanced_tests(struct pgtable_debug_args *args)
343 	unsigned long vaddr = args->vaddr;
349 	page = (args->pud_pfn != ULONG_MAX) ? pfn_to_page(args->pud_pfn) : NULL;
364 	pud = pfn_pud(args->pud_pfn, args->page_prot);
371 	set_pud_at(args->mm, vaddr, args->pudp, pud);
373 	pudp_set_wrprotect(args->mm, vaddr, args->pudp);
374 	pud = READ_ONCE(*args->pudp);
378 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);
379 	pud = READ_ONCE(*args->pudp);
382 	pud = pfn_pud(args->pud_pfn, args->page_prot);
386 	set_pud_at(args->mm, vaddr, args->pudp, pud);
390 	pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1);
391 	pud = READ_ONCE(*args->pudp);
395 	pudp_huge_get_and_clear_full(args->vma, vaddr, args->pudp, 1);
396 	pud = READ_ONCE(*args->pudp);
400 	pud = pfn_pud(args->pud_pfn, args->page_prot);
403 	set_pud_at(args->mm, vaddr, args->pudp, pud);
405 	pudp_test_and_clear_young(args->vma, vaddr, args->pudp);
406 	pud = READ_ONCE(*args->pudp);
409 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);
412 static void __init pud_leaf_tests(struct pgtable_debug_args *args)
420 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);
428 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }
429 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }
430 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }
433 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) { }
434 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }
435 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) { }
436 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }
437 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) { }
438 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }
442 static void __init pmd_huge_tests(struct pgtable_debug_args *args)
446 	if (!arch_vmap_pmd_supported(args->page_prot) ||
447 	    args->fixed_alignment < PMD_SIZE)
455 	WRITE_ONCE(*args->pmdp, __pmd(0));
456 	WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot));
457 	WARN_ON(!pmd_clear_huge(args->pmdp));
458 	pmd = READ_ONCE(*args->pmdp);
462 static void __init pud_huge_tests(struct pgtable_debug_args *args)
466 	if (!arch_vmap_pud_supported(args->page_prot) ||
467 	    args->fixed_alignment < PUD_SIZE)
475 	WRITE_ONCE(*args->pudp, __pud(0));
476 	WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot));
477 	WARN_ON(!pud_clear_huge(args->pudp));
478 	pud = READ_ONCE(*args->pudp);
482 static void __init pmd_huge_tests(struct pgtable_debug_args *args) { }
483 static void __init pud_huge_tests(struct pgtable_debug_args *args) { }
486 static void __init p4d_basic_tests(struct pgtable_debug_args *args)
495 static void __init pgd_basic_tests(struct pgtable_debug_args *args)
505 static void __init pud_clear_tests(struct pgtable_debug_args *args)
507 	pud_t pud = READ_ONCE(*args->pudp);
509 	if (mm_pmd_folded(args->mm))
514 	WRITE_ONCE(*args->pudp, pud);
515 	pud_clear(args->pudp);
516 	pud = READ_ONCE(*args->pudp);
520 static void __init pud_populate_tests(struct pgtable_debug_args *args)
524 	if (mm_pmd_folded(args->mm))
532 	pud_populate(args->mm, args->pudp, args->start_pmdp);
533 	pud = READ_ONCE(*args->pudp);
537 static void __init pud_clear_tests(struct pgtable_debug_args *args) { }
538 static void __init pud_populate_tests(struct pgtable_debug_args *args) { }
542 static void __init p4d_clear_tests(struct pgtable_debug_args *args)
544 	p4d_t p4d = READ_ONCE(*args->p4dp);
546 	if (mm_pud_folded(args->mm))
551 	WRITE_ONCE(*args->p4dp, p4d);
552 	p4d_clear(args->p4dp);
553 	p4d = READ_ONCE(*args->p4dp);
557 static void __init p4d_populate_tests(struct pgtable_debug_args *args)
561 	if (mm_pud_folded(args->mm))
569 	pud_clear(args->pudp);
570 	p4d_clear(args->p4dp);
571 	p4d_populate(args->mm, args->p4dp, args->start_pudp);
572 	p4d = READ_ONCE(*args->p4dp);
576 static void __init pgd_clear_tests(struct pgtable_debug_args *args)
578 	pgd_t pgd = READ_ONCE(*(args->pgdp));
580 	if (mm_p4d_folded(args->mm))
585 	WRITE_ONCE(*args->pgdp, pgd);
586 	pgd_clear(args->pgdp);
587 	pgd = READ_ONCE(*args->pgdp);
591 static void __init pgd_populate_tests(struct pgtable_debug_args *args)
595 	if (mm_p4d_folded(args->mm))
603 	p4d_clear(args->p4dp);
604 	pgd_clear(args->pgdp);
605 	pgd_populate(args->mm, args->pgdp, args->start_p4dp);
606 	pgd = READ_ONCE(*args->pgdp);
610 static void __init p4d_clear_tests(struct pgtable_debug_args *args) { }
611 static void __init pgd_clear_tests(struct pgtable_debug_args *args) { }
612 static void __init p4d_populate_tests(struct pgtable_debug_args *args) { }
613 static void __init pgd_populate_tests(struct pgtable_debug_args *args) { }
616 static void __init pte_clear_tests(struct pgtable_debug_args *args)
619 	pte_t pte = pfn_pte(args->pte_pfn, args->page_prot);
621 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;
633 	if (WARN_ON(!args->ptep))
639 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);
642 	ptep_clear(args->mm, args->vaddr, args->ptep);
643 	pte = ptep_get(args->ptep);
647 static void __init pmd_clear_tests(struct pgtable_debug_args *args)
649 	pmd_t pmd = READ_ONCE(*args->pmdp);
653 	WRITE_ONCE(*args->pmdp, pmd);
654 	pmd_clear(args->pmdp);
655 	pmd = READ_ONCE(*args->pmdp);
659 static void __init pmd_populate_tests(struct pgtable_debug_args *args)
668 	pmd_populate(args->mm, args->pmdp, args->start_ptep);
669 	pmd = READ_ONCE(*args->pmdp);
673 static void __init pte_special_tests(struct pgtable_debug_args *args)
675 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);
684 static void __init pte_protnone_tests(struct pgtable_debug_args *args)
686 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);
697 static void __init pmd_protnone_tests(struct pgtable_debug_args *args)
708 	pmd = pmd_mkhuge(pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none));
713 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { }
717 static void __init pte_devmap_tests(struct pgtable_debug_args *args)
719 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);
726 static void __init pmd_devmap_tests(struct pgtable_debug_args *args)
734 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
739 static void __init pud_devmap_tests(struct pgtable_debug_args *args)
747 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);
751 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }
754 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }
755 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }
758 static void __init pte_devmap_tests(struct pgtable_debug_args *args) { }
759 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }
760 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }
763 static void __init pte_soft_dirty_tests(struct pgtable_debug_args *args)
765 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);
775 static void __init pte_swap_soft_dirty_tests(struct pgtable_debug_args *args)
777 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);
788 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args)
799 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
804 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args)
816 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
821 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) { }
822 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) { }
825 static void __init pte_swap_exclusive_tests(struct pgtable_debug_args *args)
859 static void __init pte_swap_tests(struct pgtable_debug_args *args)
865 	pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);
868 	WARN_ON(args->fixed_pte_pfn != pte_pfn(pte));
872 static void __init pmd_swap_tests(struct pgtable_debug_args *args)
881 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
884 	WARN_ON(args->fixed_pmd_pfn != pmd_pfn(pmd));
887 static void __init pmd_swap_tests(struct pgtable_debug_args *args) { }
890 static void __init swap_migration_tests(struct pgtable_debug_args *args)
905 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;
931 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args)
941 	page = pfn_to_page(args->fixed_pmd_pfn);
942 	pte = mk_huge_pte(page, args->page_prot);
949 	pte = pfn_pte(args->fixed_pmd_pfn, args->page_prot);
955 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) { }
959 static void __init pmd_thp_tests(struct pgtable_debug_args *args)
978 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);
988 static void __init pud_thp_tests(struct pgtable_debug_args *args)
996 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);
1008 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }
1011 static void __init pmd_thp_tests(struct pgtable_debug_args *args) { }
1012 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }
1027 static void __init destroy_args(struct pgtable_debug_args *args)
1034 	    args->pud_pfn != ULONG_MAX) {
1035 		if (args->is_contiguous_page) {
1036 			free_contig_range(args->pud_pfn,
1039 			page = pfn_to_page(args->pud_pfn);
1043 		args->pud_pfn = ULONG_MAX;
1044 		args->pmd_pfn = ULONG_MAX;
1045 		args->pte_pfn = ULONG_MAX;
1050 	    args->pmd_pfn != ULONG_MAX) {
1051 		if (args->is_contiguous_page) {
1052 			free_contig_range(args->pmd_pfn, (1 << HPAGE_PMD_ORDER));
1054 			page = pfn_to_page(args->pmd_pfn);
1058 		args->pmd_pfn = ULONG_MAX;
1059 		args->pte_pfn = ULONG_MAX;
1062 	if (args->pte_pfn != ULONG_MAX) {
1063 		page = pfn_to_page(args->pte_pfn);
1066 		args->pte_pfn = ULONG_MAX;
1070 	if (args->start_ptep) {
1071 		pte_free(args->mm, args->start_ptep);
1072 		mm_dec_nr_ptes(args->mm);
1075 	if (args->start_pmdp) {
1076 		pmd_free(args->mm, args->start_pmdp);
1077 		mm_dec_nr_pmds(args->mm);
1080 	if (args->start_pudp) {
1081 		pud_free(args->mm, args->start_pudp);
1082 		mm_dec_nr_puds(args->mm);
1085 	if (args->start_p4dp)
1086 		p4d_free(args->mm, args->start_p4dp);
1089 	if (args->vma)
1090 		vm_area_free(args->vma);
1092 	if (args->mm)
1093 		mmdrop(args->mm);
1097 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order)
1106 			args->is_contiguous_page = true;
1143 static void __init init_fixed_pfns(struct pgtable_debug_args *args)
1165 	args->fixed_alignment = PAGE_SIZE;
1170 				 &args->fixed_alignment);
1173 		if (args->fixed_alignment == PUD_SIZE)
1180 		if (args->fixed_alignment < PMD_SIZE)
1182 					 &args->fixed_alignment);
1185 	args->fixed_pgd_pfn = __phys_to_pfn(phys & PGDIR_MASK);
1186 	args->fixed_p4d_pfn = __phys_to_pfn(phys & P4D_MASK);
1187 	args->fixed_pud_pfn = __phys_to_pfn(phys & PUD_MASK);
1188 	args->fixed_pmd_pfn = __phys_to_pfn(phys & PMD_MASK);
1189 	args->fixed_pte_pfn = __phys_to_pfn(phys & PAGE_MASK);
1190 	WARN_ON(!pfn_valid(args->fixed_pte_pfn));
1194 static int __init init_args(struct pgtable_debug_args *args)
1206 	memset(args, 0, sizeof(*args));
1207 	args->vaddr              = get_random_vaddr();
1208 	args->page_prot          = vm_get_page_prot(VM_ACCESS_FLAGS);
1209 	args->page_prot_none     = vm_get_page_prot(VM_NONE);
1210 	args->is_contiguous_page = false;
1211 	args->pud_pfn            = ULONG_MAX;
1212 	args->pmd_pfn            = ULONG_MAX;
1213 	args->pte_pfn            = ULONG_MAX;
1214 	args->fixed_pgd_pfn      = ULONG_MAX;
1215 	args->fixed_p4d_pfn      = ULONG_MAX;
1216 	args->fixed_pud_pfn      = ULONG_MAX;
1217 	args->fixed_pmd_pfn      = ULONG_MAX;
1218 	args->fixed_pte_pfn      = ULONG_MAX;
1221 	args->mm = mm_alloc();
1222 	if (!args->mm) {
1228 	args->vma = vm_area_alloc(args->mm);
1229 	if (!args->vma) {
1240 	args->pgdp = pgd_offset(args->mm, args->vaddr);
1241 	args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr);
1242 	if (!args->p4dp) {
1247 	args->start_p4dp = p4d_offset(args->pgdp, 0UL);
1248 	WARN_ON(!args->start_p4dp);
1250 	args->pudp = pud_alloc(args->mm, args->p4dp, args->vaddr);
1251 	if (!args->pudp) {
1256 	args->start_pudp = pud_offset(args->p4dp, 0UL);
1257 	WARN_ON(!args->start_pudp);
1259 	args->pmdp = pmd_alloc(args->mm, args->pudp, args->vaddr);
1260 	if (!args->pmdp) {
1265 	args->start_pmdp = pmd_offset(args->pudp, 0UL);
1266 	WARN_ON(!args->start_pmdp);
1268 	if (pte_alloc(args->mm, args->pmdp)) {
1273 	args->start_ptep = pmd_pgtable(READ_ONCE(*args->pmdp));
1274 	WARN_ON(!args->start_ptep);
1276 	init_fixed_pfns(args);
1285 		page = debug_vm_pgtable_alloc_huge_page(args,
1288 			args->pud_pfn = page_to_pfn(page);
1289 			args->pmd_pfn = args->pud_pfn;
1290 			args->pte_pfn = args->pud_pfn;
1297 		page = debug_vm_pgtable_alloc_huge_page(args, HPAGE_PMD_ORDER);
1299 			args->pmd_pfn = page_to_pfn(page);
1300 			args->pte_pfn = args->pmd_pfn;
1307 		args->pte_pfn = page_to_pfn(page);
1312 	destroy_args(args);
1318 	struct pgtable_debug_args args;
1323 	ret = init_args(&args);
1341 		pte_basic_tests(&args, idx);
1342 		pmd_basic_tests(&args, idx);
1343 		pud_basic_tests(&args, idx);
1353 	p4d_basic_tests(&args);
1354 	pgd_basic_tests(&args);
1356 	pmd_leaf_tests(&args);
1357 	pud_leaf_tests(&args);
1359 	pte_special_tests(&args);
1360 	pte_protnone_tests(&args);
1361 	pmd_protnone_tests(&args);
1363 	pte_devmap_tests(&args);
1364 	pmd_devmap_tests(&args);
1365 	pud_devmap_tests(&args);
1367 	pte_soft_dirty_tests(&args);
1368 	pmd_soft_dirty_tests(&args);
1369 	pte_swap_soft_dirty_tests(&args);
1370 	pmd_swap_soft_dirty_tests(&args);
1372 	pte_swap_exclusive_tests(&args);
1374 	pte_swap_tests(&args);
1375 	pmd_swap_tests(&args);
1377 	swap_migration_tests(&args);
1379 	pmd_thp_tests(&args);
1380 	pud_thp_tests(&args);
1382 	hugetlb_basic_tests(&args);
1389 	args.ptep = pte_offset_map_lock(args.mm, args.pmdp, args.vaddr, &ptl);
1390 	pte_clear_tests(&args);
1391 	pte_advanced_tests(&args);
1392 	if (args.ptep)
1393 		pte_unmap_unlock(args.ptep, ptl);
1395 	ptl = pmd_lock(args.mm, args.pmdp);
1396 	pmd_clear_tests(&args);
1397 	pmd_advanced_tests(&args);
1398 	pmd_huge_tests(&args);
1399 	pmd_populate_tests(&args);
1402 	ptl = pud_lock(args.mm, args.pudp);
1403 	pud_clear_tests(&args);
1404 	pud_advanced_tests(&args);
1405 	pud_huge_tests(&args);
1406 	pud_populate_tests(&args);
1409 	spin_lock(&(args.mm->page_table_lock));
1410 	p4d_clear_tests(&args);
1411 	pgd_clear_tests(&args);
1412 	p4d_populate_tests(&args);
1413 	pgd_populate_tests(&args);
1414 	spin_unlock(&(args.mm->page_table_lock));
1416 	destroy_args(&args);