1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Fault injection for both 32 and 64bit guests.
4 *
5 * Copyright (C) 2012,2013 - ARM Ltd
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 *
8 * Based on arch/arm/kvm/emulate.c
9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 */
12
13#include <hyp/adjust_pc.h>
14#include <linux/kvm_host.h>
15#include <asm/kvm_emulate.h>
16#include <asm/kvm_mmu.h>
17#include <asm/kvm_nested.h>
18
19#if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__)
20#error Hypervisor code only!
21#endif
22
23static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
24{
25	u64 val;
26
27	if (unlikely(vcpu_has_nv(vcpu)))
28		return vcpu_read_sys_reg(vcpu, reg);
29	else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
30		return val;
31
32	return __vcpu_sys_reg(vcpu, reg);
33}
34
35static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
36{
37	if (unlikely(vcpu_has_nv(vcpu)))
38		vcpu_write_sys_reg(vcpu, val, reg);
39	else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
40		__vcpu_sys_reg(vcpu, reg) = val;
41}
42
43static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode,
44			      u64 val)
45{
46	if (unlikely(vcpu_has_nv(vcpu))) {
47		if (target_mode == PSR_MODE_EL1h)
48			vcpu_write_sys_reg(vcpu, val, SPSR_EL1);
49		else
50			vcpu_write_sys_reg(vcpu, val, SPSR_EL2);
51	} else if (has_vhe()) {
52		write_sysreg_el1(val, SYS_SPSR);
53	} else {
54		__vcpu_sys_reg(vcpu, SPSR_EL1) = val;
55	}
56}
57
58static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
59{
60	if (has_vhe())
61		write_sysreg(val, spsr_abt);
62	else
63		vcpu->arch.ctxt.spsr_abt = val;
64}
65
66static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val)
67{
68	if (has_vhe())
69		write_sysreg(val, spsr_und);
70	else
71		vcpu->arch.ctxt.spsr_und = val;
72}
73
74/*
75 * This performs the exception entry at a given EL (@target_mode), stashing PC
76 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE.
77 * The EL passed to this function *must* be a non-secure, privileged mode with
78 * bit 0 being set (PSTATE.SP == 1).
79 *
80 * When an exception is taken, most PSTATE fields are left unchanged in the
81 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all
82 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx
83 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0.
84 *
85 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429.
86 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426.
87 *
88 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from
89 * MSB to LSB.
90 */
91static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
92			      enum exception_type type)
93{
94	unsigned long sctlr, vbar, old, new, mode;
95	u64 exc_offset;
96
97	mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
98
99	if      (mode == target_mode)
100		exc_offset = CURRENT_EL_SP_ELx_VECTOR;
101	else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
102		exc_offset = CURRENT_EL_SP_EL0_VECTOR;
103	else if (!(mode & PSR_MODE32_BIT))
104		exc_offset = LOWER_EL_AArch64_VECTOR;
105	else
106		exc_offset = LOWER_EL_AArch32_VECTOR;
107
108	switch (target_mode) {
109	case PSR_MODE_EL1h:
110		vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
111		sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
112		__vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
113		break;
114	case PSR_MODE_EL2h:
115		vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2);
116		sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2);
117		__vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2);
118		break;
119	default:
120		/* Don't do that */
121		BUG();
122	}
123
124	*vcpu_pc(vcpu) = vbar + exc_offset + type;
125
126	old = *vcpu_cpsr(vcpu);
127	new = 0;
128
129	new |= (old & PSR_N_BIT);
130	new |= (old & PSR_Z_BIT);
131	new |= (old & PSR_C_BIT);
132	new |= (old & PSR_V_BIT);
133
134	if (kvm_has_mte(kern_hyp_va(vcpu->kvm)))
135		new |= PSR_TCO_BIT;
136
137	new |= (old & PSR_DIT_BIT);
138
139	// PSTATE.UAO is set to zero upon any exception to AArch64
140	// See ARM DDI 0487E.a, page D5-2579.
141
142	// PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
143	// SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
144	// See ARM DDI 0487E.a, page D5-2578.
145	new |= (old & PSR_PAN_BIT);
146	if (!(sctlr & SCTLR_EL1_SPAN))
147		new |= PSR_PAN_BIT;
148
149	// PSTATE.SS is set to zero upon any exception to AArch64
150	// See ARM DDI 0487E.a, page D2-2452.
151
152	// PSTATE.IL is set to zero upon any exception to AArch64
153	// See ARM DDI 0487E.a, page D1-2306.
154
155	// PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
156	// See ARM DDI 0487E.a, page D13-3258
157	if (sctlr & SCTLR_ELx_DSSBS)
158		new |= PSR_SSBS_BIT;
159
160	// PSTATE.BTYPE is set to zero upon any exception to AArch64
161	// See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
162
163	new |= PSR_D_BIT;
164	new |= PSR_A_BIT;
165	new |= PSR_I_BIT;
166	new |= PSR_F_BIT;
167
168	new |= target_mode;
169
170	*vcpu_cpsr(vcpu) = new;
171	__vcpu_write_spsr(vcpu, target_mode, old);
172}
173
174/*
175 * When an exception is taken, most CPSR fields are left unchanged in the
176 * handler. However, some are explicitly overridden (e.g. M[4:0]).
177 *
178 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
179 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
180 * obsoleted by the ARMv7 virtualization extensions and is RES0.
181 *
182 * For the SPSR layout seen from AArch32, see:
183 * - ARM DDI 0406C.d, page B1-1148
184 * - ARM DDI 0487E.a, page G8-6264
185 *
186 * For the SPSR_ELx layout for AArch32 seen from AArch64, see:
187 * - ARM DDI 0487E.a, page C5-426
188 *
189 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
190 * MSB to LSB.
191 */
192static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
193{
194	u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
195	unsigned long old, new;
196
197	old = *vcpu_cpsr(vcpu);
198	new = 0;
199
200	new |= (old & PSR_AA32_N_BIT);
201	new |= (old & PSR_AA32_Z_BIT);
202	new |= (old & PSR_AA32_C_BIT);
203	new |= (old & PSR_AA32_V_BIT);
204	new |= (old & PSR_AA32_Q_BIT);
205
206	// CPSR.IT[7:0] are set to zero upon any exception
207	// See ARM DDI 0487E.a, section G1.12.3
208	// See ARM DDI 0406C.d, section B1.8.3
209
210	new |= (old & PSR_AA32_DIT_BIT);
211
212	// CPSR.SSBS is set to SCTLR.DSSBS upon any exception
213	// See ARM DDI 0487E.a, page G8-6244
214	if (sctlr & BIT(31))
215		new |= PSR_AA32_SSBS_BIT;
216
217	// CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
218	// SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
219	// See ARM DDI 0487E.a, page G8-6246
220	new |= (old & PSR_AA32_PAN_BIT);
221	if (!(sctlr & BIT(23)))
222		new |= PSR_AA32_PAN_BIT;
223
224	// SS does not exist in AArch32, so ignore
225
226	// CPSR.IL is set to zero upon any exception
227	// See ARM DDI 0487E.a, page G1-5527
228
229	new |= (old & PSR_AA32_GE_MASK);
230
231	// CPSR.IT[7:0] are set to zero upon any exception
232	// See prior comment above
233
234	// CPSR.E is set to SCTLR.EE upon any exception
235	// See ARM DDI 0487E.a, page G8-6245
236	// See ARM DDI 0406C.d, page B4-1701
237	if (sctlr & BIT(25))
238		new |= PSR_AA32_E_BIT;
239
240	// CPSR.A is unchanged upon an exception to Undefined, Supervisor
241	// CPSR.A is set upon an exception to other modes
242	// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
243	// See ARM DDI 0406C.d, page B1-1182
244	new |= (old & PSR_AA32_A_BIT);
245	if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
246		new |= PSR_AA32_A_BIT;
247
248	// CPSR.I is set upon any exception
249	// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
250	// See ARM DDI 0406C.d, page B1-1182
251	new |= PSR_AA32_I_BIT;
252
253	// CPSR.F is set upon an exception to FIQ
254	// CPSR.F is unchanged upon an exception to other modes
255	// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
256	// See ARM DDI 0406C.d, page B1-1182
257	new |= (old & PSR_AA32_F_BIT);
258	if (mode == PSR_AA32_MODE_FIQ)
259		new |= PSR_AA32_F_BIT;
260
261	// CPSR.T is set to SCTLR.TE upon any exception
262	// See ARM DDI 0487E.a, page G8-5514
263	// See ARM DDI 0406C.d, page B1-1181
264	if (sctlr & BIT(30))
265		new |= PSR_AA32_T_BIT;
266
267	new |= mode;
268
269	return new;
270}
271
272/*
273 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
274 */
275static const u8 return_offsets[8][2] = {
276	[0] = { 0, 0 },		/* Reset, unused */
277	[1] = { 4, 2 },		/* Undefined */
278	[2] = { 0, 0 },		/* SVC, unused */
279	[3] = { 4, 4 },		/* Prefetch abort */
280	[4] = { 8, 8 },		/* Data abort */
281	[5] = { 0, 0 },		/* HVC, unused */
282	[6] = { 4, 4 },		/* IRQ, unused */
283	[7] = { 4, 4 },		/* FIQ, unused */
284};
285
286static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
287{
288	unsigned long spsr = *vcpu_cpsr(vcpu);
289	bool is_thumb = (spsr & PSR_AA32_T_BIT);
290	u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
291	u32 return_address;
292
293	*vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
294	return_address   = *vcpu_pc(vcpu);
295	return_address  += return_offsets[vect_offset >> 2][is_thumb];
296
297	/* KVM only enters the ABT and UND modes, so only deal with those */
298	switch(mode) {
299	case PSR_AA32_MODE_ABT:
300		__vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
301		vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
302		break;
303
304	case PSR_AA32_MODE_UND:
305		__vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
306		vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
307		break;
308	}
309
310	/* Branch to exception vector */
311	if (sctlr & (1 << 13))
312		vect_offset += 0xffff0000;
313	else /* always have security exceptions */
314		vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
315
316	*vcpu_pc(vcpu) = vect_offset;
317}
318
319static void kvm_inject_exception(struct kvm_vcpu *vcpu)
320{
321	if (vcpu_el1_is_32bit(vcpu)) {
322		switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
323		case unpack_vcpu_flag(EXCEPT_AA32_UND):
324			enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
325			break;
326		case unpack_vcpu_flag(EXCEPT_AA32_IABT):
327			enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
328			break;
329		case unpack_vcpu_flag(EXCEPT_AA32_DABT):
330			enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
331			break;
332		default:
333			/* Err... */
334			break;
335		}
336	} else {
337		switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
338		case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
339			enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
340			break;
341
342		case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
343			enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
344			break;
345
346		case unpack_vcpu_flag(EXCEPT_AA64_EL2_IRQ):
347			enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
348			break;
349
350		default:
351			/*
352			 * Only EL1_SYNC and EL2_{SYNC,IRQ} makes
353			 * sense so far. Everything else gets silently
354			 * ignored.
355			 */
356			break;
357		}
358	}
359}
360
361/*
362 * Adjust the guest PC (and potentially exception state) depending on
363 * flags provided by the emulation code.
364 */
365void __kvm_adjust_pc(struct kvm_vcpu *vcpu)
366{
367	if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
368		kvm_inject_exception(vcpu);
369		vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
370		vcpu_clear_flag(vcpu, EXCEPT_MASK);
371	} else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
372		kvm_skip_instr(vcpu);
373		vcpu_clear_flag(vcpu, INCREMENT_PC);
374	}
375}
376