1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/arm-smccc.h>
8#include <linux/preempt.h>
9#include <linux/kvm_host.h>
10#include <linux/uaccess.h>
11#include <linux/wait.h>
12
13#include <asm/cputype.h>
14#include <asm/kvm_emulate.h>
15
16#include <kvm/arm_psci.h>
17#include <kvm/arm_hypercalls.h>
18
19/*
20 * This is an implementation of the Power State Coordination Interface
21 * as described in ARM document number ARM DEN 0022A.
22 */
23
24#define AFFINITY_MASK(level)	~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
25
26static unsigned long psci_affinity_mask(unsigned long affinity_level)
27{
28	if (affinity_level <= 3)
29		return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level);
30
31	return 0;
32}
33
34static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu)
35{
36	/*
37	 * NOTE: For simplicity, we make VCPU suspend emulation to be
38	 * same-as WFI (Wait-for-interrupt) emulation.
39	 *
40	 * This means for KVM the wakeup events are interrupts and
41	 * this is consistent with intended use of StateID as described
42	 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
43	 *
44	 * Further, we also treat power-down request to be same as
45	 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
46	 * specification (ARM DEN 0022A). This means all suspend states
47	 * for KVM will preserve the register state.
48	 */
49	kvm_vcpu_wfi(vcpu);
50
51	return PSCI_RET_SUCCESS;
52}
53
54static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu,
55					   unsigned long affinity)
56{
57	return !(affinity & ~MPIDR_HWID_BITMASK);
58}
59
60static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
61{
62	struct vcpu_reset_state *reset_state;
63	struct kvm *kvm = source_vcpu->kvm;
64	struct kvm_vcpu *vcpu = NULL;
65	int ret = PSCI_RET_SUCCESS;
66	unsigned long cpu_id;
67
68	cpu_id = smccc_get_arg1(source_vcpu);
69	if (!kvm_psci_valid_affinity(source_vcpu, cpu_id))
70		return PSCI_RET_INVALID_PARAMS;
71
72	vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
73
74	/*
75	 * Make sure the caller requested a valid CPU and that the CPU is
76	 * turned off.
77	 */
78	if (!vcpu)
79		return PSCI_RET_INVALID_PARAMS;
80
81	spin_lock(&vcpu->arch.mp_state_lock);
82	if (!kvm_arm_vcpu_stopped(vcpu)) {
83		if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
84			ret = PSCI_RET_ALREADY_ON;
85		else
86			ret = PSCI_RET_INVALID_PARAMS;
87
88		goto out_unlock;
89	}
90
91	reset_state = &vcpu->arch.reset_state;
92
93	reset_state->pc = smccc_get_arg2(source_vcpu);
94
95	/* Propagate caller endianness */
96	reset_state->be = kvm_vcpu_is_be(source_vcpu);
97
98	/*
99	 * NOTE: We always update r0 (or x0) because for PSCI v0.1
100	 * the general purpose registers are undefined upon CPU_ON.
101	 */
102	reset_state->r0 = smccc_get_arg3(source_vcpu);
103
104	reset_state->reset = true;
105	kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
106
107	/*
108	 * Make sure the reset request is observed if the RUNNABLE mp_state is
109	 * observed.
110	 */
111	smp_wmb();
112
113	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
114	kvm_vcpu_wake_up(vcpu);
115
116out_unlock:
117	spin_unlock(&vcpu->arch.mp_state_lock);
118	return ret;
119}
120
121static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
122{
123	int matching_cpus = 0;
124	unsigned long i, mpidr;
125	unsigned long target_affinity;
126	unsigned long target_affinity_mask;
127	unsigned long lowest_affinity_level;
128	struct kvm *kvm = vcpu->kvm;
129	struct kvm_vcpu *tmp;
130
131	target_affinity = smccc_get_arg1(vcpu);
132	lowest_affinity_level = smccc_get_arg2(vcpu);
133
134	if (!kvm_psci_valid_affinity(vcpu, target_affinity))
135		return PSCI_RET_INVALID_PARAMS;
136
137	/* Determine target affinity mask */
138	target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
139	if (!target_affinity_mask)
140		return PSCI_RET_INVALID_PARAMS;
141
142	/* Ignore other bits of target affinity */
143	target_affinity &= target_affinity_mask;
144
145	/*
146	 * If one or more VCPU matching target affinity are running
147	 * then ON else OFF
148	 */
149	kvm_for_each_vcpu(i, tmp, kvm) {
150		mpidr = kvm_vcpu_get_mpidr_aff(tmp);
151		if ((mpidr & target_affinity_mask) == target_affinity) {
152			matching_cpus++;
153			if (!kvm_arm_vcpu_stopped(tmp))
154				return PSCI_0_2_AFFINITY_LEVEL_ON;
155		}
156	}
157
158	if (!matching_cpus)
159		return PSCI_RET_INVALID_PARAMS;
160
161	return PSCI_0_2_AFFINITY_LEVEL_OFF;
162}
163
164static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
165{
166	unsigned long i;
167	struct kvm_vcpu *tmp;
168
169	/*
170	 * The KVM ABI specifies that a system event exit may call KVM_RUN
171	 * again and may perform shutdown/reboot at a later time that when the
172	 * actual request is made.  Since we are implementing PSCI and a
173	 * caller of PSCI reboot and shutdown expects that the system shuts
174	 * down or reboots immediately, let's make sure that VCPUs are not run
175	 * after this call is handled and before the VCPUs have been
176	 * re-initialized.
177	 */
178	kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
179		spin_lock(&tmp->arch.mp_state_lock);
180		WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
181		spin_unlock(&tmp->arch.mp_state_lock);
182	}
183	kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
184
185	memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
186	vcpu->run->system_event.type = type;
187	vcpu->run->system_event.ndata = 1;
188	vcpu->run->system_event.data[0] = flags;
189	vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
190}
191
192static void kvm_psci_system_off(struct kvm_vcpu *vcpu)
193{
194	kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0);
195}
196
197static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
198{
199	kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0);
200}
201
202static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu)
203{
204	kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET,
205				 KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2);
206}
207
208static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu)
209{
210	struct kvm_run *run = vcpu->run;
211
212	memset(&run->system_event, 0, sizeof(vcpu->run->system_event));
213	run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND;
214	run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
215}
216
217static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu)
218{
219	int i;
220
221	/*
222	 * Zero the input registers' upper 32 bits. They will be fully
223	 * zeroed on exit, so we're fine changing them in place.
224	 */
225	for (i = 1; i < 4; i++)
226		vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i)));
227}
228
229static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn)
230{
231	/*
232	 * Prevent 32 bit guests from calling 64 bit PSCI functions.
233	 */
234	if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu))
235		return PSCI_RET_NOT_SUPPORTED;
236
237	return 0;
238}
239
240static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
241{
242	u32 psci_fn = smccc_get_function(vcpu);
243	unsigned long val;
244	int ret = 1;
245
246	switch (psci_fn) {
247	case PSCI_0_2_FN_PSCI_VERSION:
248		/*
249		 * Bits[31:16] = Major Version = 0
250		 * Bits[15:0] = Minor Version = 2
251		 */
252		val = KVM_ARM_PSCI_0_2;
253		break;
254	case PSCI_0_2_FN_CPU_SUSPEND:
255	case PSCI_0_2_FN64_CPU_SUSPEND:
256		val = kvm_psci_vcpu_suspend(vcpu);
257		break;
258	case PSCI_0_2_FN_CPU_OFF:
259		kvm_arm_vcpu_power_off(vcpu);
260		val = PSCI_RET_SUCCESS;
261		break;
262	case PSCI_0_2_FN_CPU_ON:
263		kvm_psci_narrow_to_32bit(vcpu);
264		fallthrough;
265	case PSCI_0_2_FN64_CPU_ON:
266		val = kvm_psci_vcpu_on(vcpu);
267		break;
268	case PSCI_0_2_FN_AFFINITY_INFO:
269		kvm_psci_narrow_to_32bit(vcpu);
270		fallthrough;
271	case PSCI_0_2_FN64_AFFINITY_INFO:
272		val = kvm_psci_vcpu_affinity_info(vcpu);
273		break;
274	case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
275		/*
276		 * Trusted OS is MP hence does not require migration
277	         * or
278		 * Trusted OS is not present
279		 */
280		val = PSCI_0_2_TOS_MP;
281		break;
282	case PSCI_0_2_FN_SYSTEM_OFF:
283		kvm_psci_system_off(vcpu);
284		/*
285		 * We shouldn't be going back to guest VCPU after
286		 * receiving SYSTEM_OFF request.
287		 *
288		 * If user space accidentally/deliberately resumes
289		 * guest VCPU after SYSTEM_OFF request then guest
290		 * VCPU should see internal failure from PSCI return
291		 * value. To achieve this, we preload r0 (or x0) with
292		 * PSCI return value INTERNAL_FAILURE.
293		 */
294		val = PSCI_RET_INTERNAL_FAILURE;
295		ret = 0;
296		break;
297	case PSCI_0_2_FN_SYSTEM_RESET:
298		kvm_psci_system_reset(vcpu);
299		/*
300		 * Same reason as SYSTEM_OFF for preloading r0 (or x0)
301		 * with PSCI return value INTERNAL_FAILURE.
302		 */
303		val = PSCI_RET_INTERNAL_FAILURE;
304		ret = 0;
305		break;
306	default:
307		val = PSCI_RET_NOT_SUPPORTED;
308		break;
309	}
310
311	smccc_set_retval(vcpu, val, 0, 0, 0);
312	return ret;
313}
314
315static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor)
316{
317	unsigned long val = PSCI_RET_NOT_SUPPORTED;
318	u32 psci_fn = smccc_get_function(vcpu);
319	struct kvm *kvm = vcpu->kvm;
320	u32 arg;
321	int ret = 1;
322
323	switch(psci_fn) {
324	case PSCI_0_2_FN_PSCI_VERSION:
325		val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1;
326		break;
327	case PSCI_1_0_FN_PSCI_FEATURES:
328		arg = smccc_get_arg1(vcpu);
329		val = kvm_psci_check_allowed_function(vcpu, arg);
330		if (val)
331			break;
332
333		val = PSCI_RET_NOT_SUPPORTED;
334
335		switch(arg) {
336		case PSCI_0_2_FN_PSCI_VERSION:
337		case PSCI_0_2_FN_CPU_SUSPEND:
338		case PSCI_0_2_FN64_CPU_SUSPEND:
339		case PSCI_0_2_FN_CPU_OFF:
340		case PSCI_0_2_FN_CPU_ON:
341		case PSCI_0_2_FN64_CPU_ON:
342		case PSCI_0_2_FN_AFFINITY_INFO:
343		case PSCI_0_2_FN64_AFFINITY_INFO:
344		case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
345		case PSCI_0_2_FN_SYSTEM_OFF:
346		case PSCI_0_2_FN_SYSTEM_RESET:
347		case PSCI_1_0_FN_PSCI_FEATURES:
348		case ARM_SMCCC_VERSION_FUNC_ID:
349			val = 0;
350			break;
351		case PSCI_1_0_FN_SYSTEM_SUSPEND:
352		case PSCI_1_0_FN64_SYSTEM_SUSPEND:
353			if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
354				val = 0;
355			break;
356		case PSCI_1_1_FN_SYSTEM_RESET2:
357		case PSCI_1_1_FN64_SYSTEM_RESET2:
358			if (minor >= 1)
359				val = 0;
360			break;
361		}
362		break;
363	case PSCI_1_0_FN_SYSTEM_SUSPEND:
364		kvm_psci_narrow_to_32bit(vcpu);
365		fallthrough;
366	case PSCI_1_0_FN64_SYSTEM_SUSPEND:
367		/*
368		 * Return directly to userspace without changing the vCPU's
369		 * registers. Userspace depends on reading the SMCCC parameters
370		 * to implement SYSTEM_SUSPEND.
371		 */
372		if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
373			kvm_psci_system_suspend(vcpu);
374			return 0;
375		}
376		break;
377	case PSCI_1_1_FN_SYSTEM_RESET2:
378		kvm_psci_narrow_to_32bit(vcpu);
379		fallthrough;
380	case PSCI_1_1_FN64_SYSTEM_RESET2:
381		if (minor >= 1) {
382			arg = smccc_get_arg1(vcpu);
383
384			if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET ||
385			    arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) {
386				kvm_psci_system_reset2(vcpu);
387				vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE);
388				return 0;
389			}
390
391			val = PSCI_RET_INVALID_PARAMS;
392			break;
393		}
394		break;
395	default:
396		return kvm_psci_0_2_call(vcpu);
397	}
398
399	smccc_set_retval(vcpu, val, 0, 0, 0);
400	return ret;
401}
402
403static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
404{
405	u32 psci_fn = smccc_get_function(vcpu);
406	unsigned long val;
407
408	switch (psci_fn) {
409	case KVM_PSCI_FN_CPU_OFF:
410		kvm_arm_vcpu_power_off(vcpu);
411		val = PSCI_RET_SUCCESS;
412		break;
413	case KVM_PSCI_FN_CPU_ON:
414		val = kvm_psci_vcpu_on(vcpu);
415		break;
416	default:
417		val = PSCI_RET_NOT_SUPPORTED;
418		break;
419	}
420
421	smccc_set_retval(vcpu, val, 0, 0, 0);
422	return 1;
423}
424
425/**
426 * kvm_psci_call - handle PSCI call if r0 value is in range
427 * @vcpu: Pointer to the VCPU struct
428 *
429 * Handle PSCI calls from guests through traps from HVC instructions.
430 * The calling convention is similar to SMC calls to the secure world
431 * where the function number is placed in r0.
432 *
433 * This function returns: > 0 (success), 0 (success but exit to user
434 * space), and < 0 (errors)
435 *
436 * Errors:
437 * -EINVAL: Unrecognized PSCI function
438 */
439int kvm_psci_call(struct kvm_vcpu *vcpu)
440{
441	u32 psci_fn = smccc_get_function(vcpu);
442	int version = kvm_psci_version(vcpu);
443	unsigned long val;
444
445	val = kvm_psci_check_allowed_function(vcpu, psci_fn);
446	if (val) {
447		smccc_set_retval(vcpu, val, 0, 0, 0);
448		return 1;
449	}
450
451	switch (version) {
452	case KVM_ARM_PSCI_1_1:
453		return kvm_psci_1_x_call(vcpu, 1);
454	case KVM_ARM_PSCI_1_0:
455		return kvm_psci_1_x_call(vcpu, 0);
456	case KVM_ARM_PSCI_0_2:
457		return kvm_psci_0_2_call(vcpu);
458	case KVM_ARM_PSCI_0_1:
459		return kvm_psci_0_1_call(vcpu);
460	default:
461		WARN_ONCE(1, "Unknown PSCI version %d", version);
462		smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
463		return 1;
464	}
465}
466