kvm/mmu/mmu.c

1141  * information in the iterator may not be valid.
1168  * Must be used with a valid iterator: e.g. after rmap_get_first().
1482 static void rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator,
1485 	iterator->level = level;
1486 	iterator->gfn = iterator->start_gfn;
1487 	iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
1488 	iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
1491 static void slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1496 	iterator->slot = slot;
1497 	iterator->start_level = start_level;
1498 	iterator->end_level = end_level;
1499 	iterator->start_gfn = start_gfn;
1500 	iterator->end_gfn = end_gfn;
1502 	rmap_walk_init_level(iterator, iterator->start_level);
1505 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1507 	return !!iterator->rmap;
1510 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1512 	while (++iterator->rmap <= iterator->end_rmap) {
1513 		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1515 		if (iterator->rmap->val)
1519 	if (++iterator->level > iterator->end_level) {
1520 		iterator->rmap = NULL;
1524 	rmap_walk_init_level(iterator, iterator->level);
1542 	struct slot_rmap_walk_iterator iterator;
1546 				 range->start, range->end - 1, &iterator)
1547 		ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
1548 			       iterator.level);
2323 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
2327 	iterator->addr = addr;
2328 	iterator->shadow_addr = root;
2329 	iterator->level = vcpu->arch.mmu->root_role.level;
2331 	if (iterator->level >= PT64_ROOT_4LEVEL &&
2334 		iterator->level = PT32E_ROOT_LEVEL;
2336 	if (iterator->level == PT32E_ROOT_LEVEL) {
2343 		iterator->shadow_addr
2345 		iterator->shadow_addr &= SPTE_BASE_ADDR_MASK;
2346 		--iterator->level;
2347 		if (!iterator->shadow_addr)
2348 			iterator->level = 0;
2352 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2355 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,
2359 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2361 	if (iterator->level < PG_LEVEL_4K)
2364 	iterator->index = SPTE_INDEX(iterator->addr, iterator->level);
2365 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2369 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2372 	if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
2373 		iterator->level = 0;
2377 	iterator->shadow_addr = spte & SPTE_BASE_ADDR_MASK;
2378 	--iterator->level;
2381 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2383 	__shadow_walk_next(iterator, *iterator->sptep);
3396 	struct kvm_shadow_walk_iterator iterator;
3400 	for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
3401 		sptep = iterator.sptep;
4087 	struct kvm_shadow_walk_iterator iterator;
4091 	for (shadow_walk_init(&iterator, vcpu, addr),
4092 	     *root_level = iterator.level;
4093 	     shadow_walk_okay(&iterator);
4094 	     __shadow_walk_next(&iterator, spte)) {
4095 		leaf = iterator.level;
4096 		spte = mmu_spte_get_lockless(iterator.sptep);
4209 	struct kvm_shadow_walk_iterator iterator;
4213 	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
4214 		clear_sp_write_flooding_count(iterator.sptep);
5927 	struct kvm_shadow_walk_iterator iterator;
5943 	for_each_shadow_entry_using_root(vcpu, root_hpa, addr, iterator) {
5944 		struct kvm_mmu_page *sp = sptep_to_sp(iterator.sptep);
5947 			int ret = kvm_sync_spte(vcpu, sp, iterator.index);
5950 				mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);
5952 				kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep);
6072 	struct slot_rmap_walk_iterator iterator;
6077 			end_gfn, &iterator) {
6078 		if (iterator.rmap)
6079 			flush |= fn(kvm, iterator.rmap, slot);
6084 							    iterator.gfn - start_gfn + 1);
6600 		 * rmap iterator should be restarted because the MMU lock was
6648 		 * lock was dropped. Either way, restart the iterator to get it