kvm/hyp/pgtable.c

141 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
143 	return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
151 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
153 	kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
209 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level);
212 				      struct kvm_pgtable_mm_ops *mm_ops,
221 		.mm_ops	= mm_ops,
262 	childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
263 	ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
278 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level)
293 		ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
316 		ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
457 		ctx->mm_ops->get_page(ctx->ptep);
470 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
478 	childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
482 	new = kvm_init_table_pte(childp, mm_ops);
483 	mm_ops->get_page(ctx->ptep);
518 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
524 		childp = kvm_pte_follow(ctx->old, mm_ops);
526 		if (mm_ops->page_count(childp) != 1)
544 	mm_ops->put_page(ctx->ptep);
547 		mm_ops->put_page(childp);
561 	if (!pgt->mm_ops->page_count)
569 			 struct kvm_pgtable_mm_ops *mm_ops)
578 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
584 	pgt->mm_ops		= mm_ops;
594 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
599 	mm_ops->put_page(ctx->ptep);
602 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
615 	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
827 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
858 		mm_ops->put_page(ctx->ptep);
865 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
870 		mm_ops->get_page(ctx->ptep);
890 				struct kvm_pgtable_mm_ops *mm_ops)
911 	mm_ops->put_page(ctx->ptep);
963 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
989 				mm_ops->put_page(ctx->ptep);
991 				mm_ops->get_page(ctx->ptep);
1001 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&
1003 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),
1006 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&
1008 		mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);
1018 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1019 	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
1029 	mm_ops->free_unlinked_table(childp, ctx->level);
1036 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1050 	childp = mm_ops->zalloc_page(data->memcache);
1055 		mm_ops->put_page(childp);
1064 	new = kvm_init_table_pte(childp, mm_ops);
1155 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1162 			mm_ops->put_page(ctx->ptep);
1168 		childp = kvm_pte_follow(ctx->old, mm_ops);
1170 		if (mm_ops->page_count(childp) != 1)
1181 	stage2_unmap_put_pte(ctx, mmu, mm_ops);
1183 	if (need_flush && mm_ops->dcache_clean_inval_poc)
1184 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1188 		mm_ops->put_page(childp);
1222 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1242 		if (mm_ops->icache_inval_pou &&
1244 			mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
1386 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1391 	if (mm_ops->dcache_clean_inval_poc)
1392 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1439 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1450 	pgtable = mm_ops->zalloc_page(mc);
1454 	ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,
1457 		kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);
1488 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1539 		kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);
1548 	new = kvm_init_table_pte(childp, mm_ops);
1567 			      struct kvm_pgtable_mm_ops *mm_ops,
1578 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
1584 	pgt->mm_ops		= mm_ops;
1606 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1611 	mm_ops->put_page(ctx->ptep);
1614 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
1630 	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);
1634 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level)
1654 	WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));
1656 	WARN_ON(mm_ops->page_count(pgtable) != 1);
1657 	mm_ops->put_page(pgtable);