1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <hyp/sysreg-sr.h>
8
9#include <linux/compiler.h>
10#include <linux/kvm_host.h>
11
12#include <asm/kprobes.h>
13#include <asm/kvm_asm.h>
14#include <asm/kvm_emulate.h>
15#include <asm/kvm_hyp.h>
16#include <asm/kvm_nested.h>
17
18/*
19 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
20 * pstate, which are handled as part of the el2 return state) on every
21 * switch (sp_el0 is being dealt with in the assembly code).
22 * tpidr_el0 and tpidrro_el0 only need to be switched when going
23 * to host userspace or a different VCPU.  EL1 registers only need to be
24 * switched when potentially going to run a different VCPU.  The latter two
25 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
26 */
27
28void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
29{
30	__sysreg_save_common_state(ctxt);
31}
32NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
33
34void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
35{
36	__sysreg_save_common_state(ctxt);
37	__sysreg_save_el2_return_state(ctxt);
38}
39NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
40
41void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
42{
43	__sysreg_restore_common_state(ctxt);
44}
45NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
46
47void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
48{
49	__sysreg_restore_common_state(ctxt);
50	__sysreg_restore_el2_return_state(ctxt);
51}
52NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
53
54/**
55 * __vcpu_load_switch_sysregs - Load guest system registers to the physical CPU
56 *
57 * @vcpu: The VCPU pointer
58 *
59 * Load system registers that do not affect the host's execution, for
60 * example EL1 system registers on a VHE system where the host kernel
61 * runs at EL2.  This function is called from KVM's vcpu_load() function
62 * and loading system register state early avoids having to load them on
63 * every entry to the VM.
64 */
65void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu)
66{
67	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
68	struct kvm_cpu_context *host_ctxt;
69
70	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
71	__sysreg_save_user_state(host_ctxt);
72
73	/*
74	 * When running a normal EL1 guest, we only load a new vcpu
75	 * after a context switch, which imvolves a DSB, so all
76	 * speculative EL1&0 walks will have already completed.
77	 * If running NV, the vcpu may transition between vEL1 and
78	 * vEL2 without a context switch, so make sure we complete
79	 * those walks before loading a new context.
80	 */
81	if (vcpu_has_nv(vcpu))
82		dsb(nsh);
83
84	/*
85	 * Load guest EL1 and user state
86	 *
87	 * We must restore the 32-bit state before the sysregs, thanks
88	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
89	 */
90	__sysreg32_restore_state(vcpu);
91	__sysreg_restore_user_state(guest_ctxt);
92	__sysreg_restore_el1_state(guest_ctxt);
93
94	vcpu_set_flag(vcpu, SYSREGS_ON_CPU);
95}
96
97/**
98 * __vcpu_put_switch_sysregs - Restore host system registers to the physical CPU
99 *
100 * @vcpu: The VCPU pointer
101 *
102 * Save guest system registers that do not affect the host's execution, for
103 * example EL1 system registers on a VHE system where the host kernel
104 * runs at EL2.  This function is called from KVM's vcpu_put() function
105 * and deferring saving system register state until we're no longer running the
106 * VCPU avoids having to save them on every exit from the VM.
107 */
108void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu)
109{
110	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
111	struct kvm_cpu_context *host_ctxt;
112
113	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
114
115	__sysreg_save_el1_state(guest_ctxt);
116	__sysreg_save_user_state(guest_ctxt);
117	__sysreg32_save_state(vcpu);
118
119	/* Restore host user state */
120	__sysreg_restore_user_state(host_ctxt);
121
122	vcpu_clear_flag(vcpu, SYSREGS_ON_CPU);
123}
124