1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2022 Ventana Micro Systems Inc.
4 */
5
6#include <linux/bitmap.h>
7#include <linux/cpumask.h>
8#include <linux/errno.h>
9#include <linux/err.h>
10#include <linux/module.h>
11#include <linux/smp.h>
12#include <linux/kvm_host.h>
13#include <asm/cacheflush.h>
14#include <asm/csr.h>
15#include <asm/cpufeature.h>
16#include <asm/insn-def.h>
17
18#define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
19
20void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
21					  gpa_t gpa, gpa_t gpsz,
22					  unsigned long order)
23{
24	gpa_t pos;
25
26	if (PTRS_PER_PTE < (gpsz >> order)) {
27		kvm_riscv_local_hfence_gvma_vmid_all(vmid);
28		return;
29	}
30
31	if (has_svinval()) {
32		asm volatile (SFENCE_W_INVAL() ::: "memory");
33		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
34			asm volatile (HINVAL_GVMA(%0, %1)
35			: : "r" (pos >> 2), "r" (vmid) : "memory");
36		asm volatile (SFENCE_INVAL_IR() ::: "memory");
37	} else {
38		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
39			asm volatile (HFENCE_GVMA(%0, %1)
40			: : "r" (pos >> 2), "r" (vmid) : "memory");
41	}
42}
43
44void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)
45{
46	asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory");
47}
48
49void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
50				     unsigned long order)
51{
52	gpa_t pos;
53
54	if (PTRS_PER_PTE < (gpsz >> order)) {
55		kvm_riscv_local_hfence_gvma_all();
56		return;
57	}
58
59	if (has_svinval()) {
60		asm volatile (SFENCE_W_INVAL() ::: "memory");
61		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
62			asm volatile(HINVAL_GVMA(%0, zero)
63			: : "r" (pos >> 2) : "memory");
64		asm volatile (SFENCE_INVAL_IR() ::: "memory");
65	} else {
66		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
67			asm volatile(HFENCE_GVMA(%0, zero)
68			: : "r" (pos >> 2) : "memory");
69	}
70}
71
72void kvm_riscv_local_hfence_gvma_all(void)
73{
74	asm volatile(HFENCE_GVMA(zero, zero) : : : "memory");
75}
76
77void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
78					  unsigned long asid,
79					  unsigned long gva,
80					  unsigned long gvsz,
81					  unsigned long order)
82{
83	unsigned long pos, hgatp;
84
85	if (PTRS_PER_PTE < (gvsz >> order)) {
86		kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
87		return;
88	}
89
90	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
91
92	if (has_svinval()) {
93		asm volatile (SFENCE_W_INVAL() ::: "memory");
94		for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
95			asm volatile(HINVAL_VVMA(%0, %1)
96			: : "r" (pos), "r" (asid) : "memory");
97		asm volatile (SFENCE_INVAL_IR() ::: "memory");
98	} else {
99		for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
100			asm volatile(HFENCE_VVMA(%0, %1)
101			: : "r" (pos), "r" (asid) : "memory");
102	}
103
104	csr_write(CSR_HGATP, hgatp);
105}
106
107void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
108					  unsigned long asid)
109{
110	unsigned long hgatp;
111
112	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
113
114	asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory");
115
116	csr_write(CSR_HGATP, hgatp);
117}
118
119void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
120				     unsigned long gva, unsigned long gvsz,
121				     unsigned long order)
122{
123	unsigned long pos, hgatp;
124
125	if (PTRS_PER_PTE < (gvsz >> order)) {
126		kvm_riscv_local_hfence_vvma_all(vmid);
127		return;
128	}
129
130	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
131
132	if (has_svinval()) {
133		asm volatile (SFENCE_W_INVAL() ::: "memory");
134		for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
135			asm volatile(HINVAL_VVMA(%0, zero)
136			: : "r" (pos) : "memory");
137		asm volatile (SFENCE_INVAL_IR() ::: "memory");
138	} else {
139		for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
140			asm volatile(HFENCE_VVMA(%0, zero)
141			: : "r" (pos) : "memory");
142	}
143
144	csr_write(CSR_HGATP, hgatp);
145}
146
147void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
148{
149	unsigned long hgatp;
150
151	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
152
153	asm volatile(HFENCE_VVMA(zero, zero) : : : "memory");
154
155	csr_write(CSR_HGATP, hgatp);
156}
157
158void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
159{
160	unsigned long vmid;
161
162	if (!kvm_riscv_gstage_vmid_bits() ||
163	    vcpu->arch.last_exit_cpu == vcpu->cpu)
164		return;
165
166	/*
167	 * On RISC-V platforms with hardware VMID support, we share same
168	 * VMID for all VCPUs of a particular Guest/VM. This means we might
169	 * have stale G-stage TLB entries on the current Host CPU due to
170	 * some other VCPU of the same Guest which ran previously on the
171	 * current Host CPU.
172	 *
173	 * To cleanup stale TLB entries, we simply flush all G-stage TLB
174	 * entries by VMID whenever underlying Host CPU changes for a VCPU.
175	 */
176
177	vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
178	kvm_riscv_local_hfence_gvma_vmid_all(vmid);
179}
180
181void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
182{
183	kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);
184	local_flush_icache_all();
185}
186
187void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
188{
189	struct kvm_vmid *vmid;
190
191	vmid = &vcpu->kvm->arch.vmid;
192	kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid));
193}
194
195void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu)
196{
197	struct kvm_vmid *vmid;
198
199	vmid = &vcpu->kvm->arch.vmid;
200	kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid));
201}
202
203static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu,
204				struct kvm_riscv_hfence *out_data)
205{
206	bool ret = false;
207	struct kvm_vcpu_arch *varch = &vcpu->arch;
208
209	spin_lock(&varch->hfence_lock);
210
211	if (varch->hfence_queue[varch->hfence_head].type) {
212		memcpy(out_data, &varch->hfence_queue[varch->hfence_head],
213		       sizeof(*out_data));
214		varch->hfence_queue[varch->hfence_head].type = 0;
215
216		varch->hfence_head++;
217		if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE)
218			varch->hfence_head = 0;
219
220		ret = true;
221	}
222
223	spin_unlock(&varch->hfence_lock);
224
225	return ret;
226}
227
228static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu,
229				const struct kvm_riscv_hfence *data)
230{
231	bool ret = false;
232	struct kvm_vcpu_arch *varch = &vcpu->arch;
233
234	spin_lock(&varch->hfence_lock);
235
236	if (!varch->hfence_queue[varch->hfence_tail].type) {
237		memcpy(&varch->hfence_queue[varch->hfence_tail],
238		       data, sizeof(*data));
239
240		varch->hfence_tail++;
241		if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE)
242			varch->hfence_tail = 0;
243
244		ret = true;
245	}
246
247	spin_unlock(&varch->hfence_lock);
248
249	return ret;
250}
251
252void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
253{
254	struct kvm_riscv_hfence d = { 0 };
255	struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
256
257	while (vcpu_hfence_dequeue(vcpu, &d)) {
258		switch (d.type) {
259		case KVM_RISCV_HFENCE_UNKNOWN:
260			break;
261		case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
262			kvm_riscv_local_hfence_gvma_vmid_gpa(
263						READ_ONCE(v->vmid),
264						d.addr, d.size, d.order);
265			break;
266		case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
267			kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
268			kvm_riscv_local_hfence_vvma_asid_gva(
269						READ_ONCE(v->vmid), d.asid,
270						d.addr, d.size, d.order);
271			break;
272		case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
273			kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
274			kvm_riscv_local_hfence_vvma_asid_all(
275						READ_ONCE(v->vmid), d.asid);
276			break;
277		case KVM_RISCV_HFENCE_VVMA_GVA:
278			kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
279			kvm_riscv_local_hfence_vvma_gva(
280						READ_ONCE(v->vmid),
281						d.addr, d.size, d.order);
282			break;
283		default:
284			break;
285		}
286	}
287}
288
289static void make_xfence_request(struct kvm *kvm,
290				unsigned long hbase, unsigned long hmask,
291				unsigned int req, unsigned int fallback_req,
292				const struct kvm_riscv_hfence *data)
293{
294	unsigned long i;
295	struct kvm_vcpu *vcpu;
296	unsigned int actual_req = req;
297	DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
298
299	bitmap_zero(vcpu_mask, KVM_MAX_VCPUS);
300	kvm_for_each_vcpu(i, vcpu, kvm) {
301		if (hbase != -1UL) {
302			if (vcpu->vcpu_id < hbase)
303				continue;
304			if (!(hmask & (1UL << (vcpu->vcpu_id - hbase))))
305				continue;
306		}
307
308		bitmap_set(vcpu_mask, i, 1);
309
310		if (!data || !data->type)
311			continue;
312
313		/*
314		 * Enqueue hfence data to VCPU hfence queue. If we don't
315		 * have space in the VCPU hfence queue then fallback to
316		 * a more conservative hfence request.
317		 */
318		if (!vcpu_hfence_enqueue(vcpu, data))
319			actual_req = fallback_req;
320	}
321
322	kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
323}
324
325void kvm_riscv_fence_i(struct kvm *kvm,
326		       unsigned long hbase, unsigned long hmask)
327{
328	make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I,
329			    KVM_REQ_FENCE_I, NULL);
330}
331
332void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
333				    unsigned long hbase, unsigned long hmask,
334				    gpa_t gpa, gpa_t gpsz,
335				    unsigned long order)
336{
337	struct kvm_riscv_hfence data;
338
339	data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
340	data.asid = 0;
341	data.addr = gpa;
342	data.size = gpsz;
343	data.order = order;
344	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
345			    KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
346}
347
348void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
349				    unsigned long hbase, unsigned long hmask)
350{
351	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
352			    KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
353}
354
355void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
356				    unsigned long hbase, unsigned long hmask,
357				    unsigned long gva, unsigned long gvsz,
358				    unsigned long order, unsigned long asid)
359{
360	struct kvm_riscv_hfence data;
361
362	data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
363	data.asid = asid;
364	data.addr = gva;
365	data.size = gvsz;
366	data.order = order;
367	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
368			    KVM_REQ_HFENCE_VVMA_ALL, &data);
369}
370
371void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
372				    unsigned long hbase, unsigned long hmask,
373				    unsigned long asid)
374{
375	struct kvm_riscv_hfence data;
376
377	data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
378	data.asid = asid;
379	data.addr = data.size = data.order = 0;
380	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
381			    KVM_REQ_HFENCE_VVMA_ALL, &data);
382}
383
384void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
385			       unsigned long hbase, unsigned long hmask,
386			       unsigned long gva, unsigned long gvsz,
387			       unsigned long order)
388{
389	struct kvm_riscv_hfence data;
390
391	data.type = KVM_RISCV_HFENCE_VVMA_GVA;
392	data.asid = 0;
393	data.addr = gva;
394	data.size = gvsz;
395	data.order = order;
396	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
397			    KVM_REQ_HFENCE_VVMA_ALL, &data);
398}
399
400void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
401			       unsigned long hbase, unsigned long hmask)
402{
403	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
404			    KVM_REQ_HFENCE_VVMA_ALL, NULL);
405}
406