1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * KVM dirty ring implementation 4 * 5 * Copyright 2019 Red Hat, Inc. 6 */ 7#include <linux/kvm_host.h> 8#include <linux/kvm.h> 9#include <linux/vmalloc.h> 10#include <linux/kvm_dirty_ring.h> 11#include <trace/events/kvm.h> 12#include "kvm_mm.h" 13 14int __weak kvm_cpu_dirty_log_size(void) 15{ 16 return 0; 17} 18 19u32 kvm_dirty_ring_get_rsvd_entries(void) 20{ 21 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size(); 22} 23 24bool kvm_use_dirty_bitmap(struct kvm *kvm) 25{ 26 lockdep_assert_held(&kvm->slots_lock); 27 28 return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap; 29} 30 31#ifndef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP 32bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm) 33{ 34 return false; 35} 36#endif 37 38static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring) 39{ 40 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index); 41} 42 43static bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring) 44{ 45 return kvm_dirty_ring_used(ring) >= ring->soft_limit; 46} 47 48static bool kvm_dirty_ring_full(struct kvm_dirty_ring *ring) 49{ 50 return kvm_dirty_ring_used(ring) >= ring->size; 51} 52 53static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) 54{ 55 struct kvm_memory_slot *memslot; 56 int as_id, id; 57 58 as_id = slot >> 16; 59 id = (u16)slot; 60 61 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS) 62 return; 63 64 memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id); 65 66 if (!memslot || (offset + __fls(mask)) >= memslot->npages) 67 return; 68 69 KVM_MMU_LOCK(kvm); 70 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask); 71 KVM_MMU_UNLOCK(kvm); 72} 73 74int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size) 75{ 76 ring->dirty_gfns = vzalloc(size); 77 if (!ring->dirty_gfns) 78 return -ENOMEM; 79 80 ring->size = size / sizeof(struct kvm_dirty_gfn); 81 ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries(); 82 ring->dirty_index = 0; 83 ring->reset_index = 0; 84 ring->index = index; 85 86 return 0; 87} 88 89static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn) 90{ 91 smp_store_release(&gfn->flags, 0); 92} 93 94static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn) 95{ 96 gfn->flags = KVM_DIRTY_GFN_F_DIRTY; 97} 98 99static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn) 100{ 101 return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET; 102} 103 104int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring) 105{ 106 u32 cur_slot, next_slot; 107 u64 cur_offset, next_offset; 108 unsigned long mask; 109 int count = 0; 110 struct kvm_dirty_gfn *entry; 111 bool first_round = true; 112 113 /* This is only needed to make compilers happy */ 114 cur_slot = cur_offset = mask = 0; 115 116 while (true) { 117 entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)]; 118 119 if (!kvm_dirty_gfn_harvested(entry)) 120 break; 121 122 next_slot = READ_ONCE(entry->slot); 123 next_offset = READ_ONCE(entry->offset); 124 125 /* Update the flags to reflect that this GFN is reset */ 126 kvm_dirty_gfn_set_invalid(entry); 127 128 ring->reset_index++; 129 count++; 130 /* 131 * Try to coalesce the reset operations when the guest is 132 * scanning pages in the same slot. 133 */ 134 if (!first_round && next_slot == cur_slot) { 135 s64 delta = next_offset - cur_offset; 136 137 if (delta >= 0 && delta < BITS_PER_LONG) { 138 mask |= 1ull << delta; 139 continue; 140 } 141 142 /* Backwards visit, careful about overflows! */ 143 if (delta > -BITS_PER_LONG && delta < 0 && 144 (mask << -delta >> -delta) == mask) { 145 cur_offset = next_offset; 146 mask = (mask << -delta) | 1; 147 continue; 148 } 149 } 150 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask); 151 cur_slot = next_slot; 152 cur_offset = next_offset; 153 mask = 1; 154 first_round = false; 155 } 156 157 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask); 158 159 /* 160 * The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared 161 * by the VCPU thread next time when it enters the guest. 162 */ 163 164 trace_kvm_dirty_ring_reset(ring); 165 166 return count; 167} 168 169void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset) 170{ 171 struct kvm_dirty_ring *ring = &vcpu->dirty_ring; 172 struct kvm_dirty_gfn *entry; 173 174 /* It should never get full */ 175 WARN_ON_ONCE(kvm_dirty_ring_full(ring)); 176 177 entry = &ring->dirty_gfns[ring->dirty_index & (ring->size - 1)]; 178 179 entry->slot = slot; 180 entry->offset = offset; 181 /* 182 * Make sure the data is filled in before we publish this to 183 * the userspace program. There's no paired kernel-side reader. 184 */ 185 smp_wmb(); 186 kvm_dirty_gfn_set_dirtied(entry); 187 ring->dirty_index++; 188 trace_kvm_dirty_ring_push(ring, slot, offset); 189 190 if (kvm_dirty_ring_soft_full(ring)) 191 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu); 192} 193 194bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu) 195{ 196 /* 197 * The VCPU isn't runnable when the dirty ring becomes soft full. 198 * The KVM_REQ_DIRTY_RING_SOFT_FULL event is always set to prevent 199 * the VCPU from running until the dirty pages are harvested and 200 * the dirty ring is reset by userspace. 201 */ 202 if (kvm_check_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu) && 203 kvm_dirty_ring_soft_full(&vcpu->dirty_ring)) { 204 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu); 205 vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL; 206 trace_kvm_dirty_ring_exit(vcpu); 207 return true; 208 } 209 210 return false; 211} 212 213struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset) 214{ 215 return vmalloc_to_page((void *)ring->dirty_gfns + offset * PAGE_SIZE); 216} 217 218void kvm_dirty_ring_free(struct kvm_dirty_ring *ring) 219{ 220 vfree(ring->dirty_gfns); 221 ring->dirty_gfns = NULL; 222} 223