1/*	$NetBSD: nvmm_x86.c,v 1.23 2022/10/06 00:22:16 msaitoh Exp $	*/
2
3/*
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the NVMM hypervisor.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32__KERNEL_RCSID(0, "$NetBSD: nvmm_x86.c,v 1.23 2022/10/06 00:22:16 msaitoh Exp $");
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/kernel.h>
37#include <sys/cpu.h>
38
39#include <uvm/uvm_extern.h>
40
41#include <x86/cputypes.h>
42#include <x86/pat.h>
43#include <x86/specialreg.h>
44
45#include <dev/nvmm/nvmm.h>
46#include <dev/nvmm/nvmm_internal.h>
47#include <dev/nvmm/x86/nvmm_x86.h>
48
49/*
50 * Code shared between x86-SVM and x86-VMX.
51 */
52
53const struct nvmm_x64_state nvmm_x86_reset_state = {
54	.segs = {
55		[NVMM_X64_SEG_ES] = {
56			.selector = 0x0000,
57			.base = 0x00000000,
58			.limit = 0xFFFF,
59			.attrib = {
60				.type = 3,
61				.s = 1,
62				.p = 1,
63			}
64		},
65		[NVMM_X64_SEG_CS] = {
66			.selector = 0xF000,
67			.base = 0xFFFF0000,
68			.limit = 0xFFFF,
69			.attrib = {
70				.type = 3,
71				.s = 1,
72				.p = 1,
73			}
74		},
75		[NVMM_X64_SEG_SS] = {
76			.selector = 0x0000,
77			.base = 0x00000000,
78			.limit = 0xFFFF,
79			.attrib = {
80				.type = 3,
81				.s = 1,
82				.p = 1,
83			}
84		},
85		[NVMM_X64_SEG_DS] = {
86			.selector = 0x0000,
87			.base = 0x00000000,
88			.limit = 0xFFFF,
89			.attrib = {
90				.type = 3,
91				.s = 1,
92				.p = 1,
93			}
94		},
95		[NVMM_X64_SEG_FS] = {
96			.selector = 0x0000,
97			.base = 0x00000000,
98			.limit = 0xFFFF,
99			.attrib = {
100				.type = 3,
101				.s = 1,
102				.p = 1,
103			}
104		},
105		[NVMM_X64_SEG_GS] = {
106			.selector = 0x0000,
107			.base = 0x00000000,
108			.limit = 0xFFFF,
109			.attrib = {
110				.type = 3,
111				.s = 1,
112				.p = 1,
113			}
114		},
115		[NVMM_X64_SEG_GDT] = {
116			.selector = 0x0000,
117			.base = 0x00000000,
118			.limit = 0xFFFF,
119			.attrib = {
120				.type = 2,
121				.s = 1,
122				.p = 1,
123			}
124		},
125		[NVMM_X64_SEG_IDT] = {
126			.selector = 0x0000,
127			.base = 0x00000000,
128			.limit = 0xFFFF,
129			.attrib = {
130				.type = 2,
131				.s = 1,
132				.p = 1,
133			}
134		},
135		[NVMM_X64_SEG_LDT] = {
136			.selector = 0x0000,
137			.base = 0x00000000,
138			.limit = 0xFFFF,
139			.attrib = {
140				.type = SDT_SYSLDT,
141				.s = 0,
142				.p = 1,
143			}
144		},
145		[NVMM_X64_SEG_TR] = {
146			.selector = 0x0000,
147			.base = 0x00000000,
148			.limit = 0xFFFF,
149			.attrib = {
150				.type = SDT_SYS286BSY,
151				.s = 0,
152				.p = 1,
153			}
154		},
155	},
156
157	.gprs = {
158		[NVMM_X64_GPR_RAX] = 0x00000000,
159		[NVMM_X64_GPR_RCX] = 0x00000000,
160		[NVMM_X64_GPR_RDX] = 0x00000600,
161		[NVMM_X64_GPR_RBX] = 0x00000000,
162		[NVMM_X64_GPR_RSP] = 0x00000000,
163		[NVMM_X64_GPR_RBP] = 0x00000000,
164		[NVMM_X64_GPR_RSI] = 0x00000000,
165		[NVMM_X64_GPR_RDI] = 0x00000000,
166		[NVMM_X64_GPR_R8] = 0x00000000,
167		[NVMM_X64_GPR_R9] = 0x00000000,
168		[NVMM_X64_GPR_R10] = 0x00000000,
169		[NVMM_X64_GPR_R11] = 0x00000000,
170		[NVMM_X64_GPR_R12] = 0x00000000,
171		[NVMM_X64_GPR_R13] = 0x00000000,
172		[NVMM_X64_GPR_R14] = 0x00000000,
173		[NVMM_X64_GPR_R15] = 0x00000000,
174		[NVMM_X64_GPR_RIP] = 0x0000FFF0,
175		[NVMM_X64_GPR_RFLAGS] = 0x00000002,
176	},
177
178	.crs = {
179		[NVMM_X64_CR_CR0] = 0x60000010,
180		[NVMM_X64_CR_CR2] = 0x00000000,
181		[NVMM_X64_CR_CR3] = 0x00000000,
182		[NVMM_X64_CR_CR4] = 0x00000000,
183		[NVMM_X64_CR_CR8] = 0x00000000,
184		[NVMM_X64_CR_XCR0] = 0x00000001,
185	},
186
187	.drs = {
188		[NVMM_X64_DR_DR0] = 0x00000000,
189		[NVMM_X64_DR_DR1] = 0x00000000,
190		[NVMM_X64_DR_DR2] = 0x00000000,
191		[NVMM_X64_DR_DR3] = 0x00000000,
192		[NVMM_X64_DR_DR6] = 0xFFFF0FF0,
193		[NVMM_X64_DR_DR7] = 0x00000400,
194	},
195
196	.msrs = {
197		[NVMM_X64_MSR_EFER] = 0x00000000,
198		[NVMM_X64_MSR_STAR] = 0x00000000,
199		[NVMM_X64_MSR_LSTAR] = 0x00000000,
200		[NVMM_X64_MSR_CSTAR] = 0x00000000,
201		[NVMM_X64_MSR_SFMASK] = 0x00000000,
202		[NVMM_X64_MSR_KERNELGSBASE] = 0x00000000,
203		[NVMM_X64_MSR_SYSENTER_CS] = 0x00000000,
204		[NVMM_X64_MSR_SYSENTER_ESP] = 0x00000000,
205		[NVMM_X64_MSR_SYSENTER_EIP] = 0x00000000,
206		[NVMM_X64_MSR_PAT] =
207		    PATENTRY(0, PAT_WB) | PATENTRY(1, PAT_WT) |
208		    PATENTRY(2, PAT_UCMINUS) | PATENTRY(3, PAT_UC) |
209		    PATENTRY(4, PAT_WB) | PATENTRY(5, PAT_WT) |
210		    PATENTRY(6, PAT_UCMINUS) | PATENTRY(7, PAT_UC),
211		[NVMM_X64_MSR_TSC] = 0,
212	},
213
214	.intr = {
215		.int_shadow = 0,
216		.int_window_exiting = 0,
217		.nmi_window_exiting = 0,
218		.evt_pending = 0,
219	},
220
221	.fpu = {
222		.fx_cw = 0x0040,
223		.fx_sw = 0x0000,
224		.fx_tw = 0x55,
225		.fx_zero = 0x55,
226		.fx_mxcsr = 0x1F80,
227	}
228};
229
230const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000001 = {
231	.eax = ~0,
232	.ebx = ~0,
233	.ecx =
234	    CPUID2_SSE3 |
235	    CPUID2_PCLMULQDQ |
236	    /* CPUID2_DTES64 excluded */
237	    /* CPUID2_MONITOR excluded */
238	    /* CPUID2_DS_CPL excluded */
239	    /* CPUID2_VMX excluded */
240	    /* CPUID2_SMX excluded */
241	    /* CPUID2_EST excluded */
242	    /* CPUID2_TM2 excluded */
243	    CPUID2_SSSE3 |
244	    /* CPUID2_CNXTID excluded */
245	    /* CPUID2_SDBG excluded */
246	    CPUID2_FMA |
247	    CPUID2_CX16 |
248	    /* CPUID2_XTPR excluded */
249	    /* CPUID2_PDCM excluded */
250	    /* CPUID2_PCID excluded, but re-included in VMX */
251	    /* CPUID2_DCA excluded */
252	    CPUID2_SSE41 |
253	    CPUID2_SSE42 |
254	    /* CPUID2_X2APIC excluded */
255	    CPUID2_MOVBE |
256	    CPUID2_POPCNT |
257	    /* CPUID2_DEADLINE excluded */
258	    CPUID2_AESNI |
259	    CPUID2_XSAVE |
260	    CPUID2_OSXSAVE |
261	    /* CPUID2_AVX excluded */
262	    CPUID2_F16C |
263	    CPUID2_RDRAND,
264	    /* CPUID2_RAZ excluded */
265	.edx =
266	    CPUID_FPU |
267	    CPUID_VME |
268	    CPUID_DE |
269	    CPUID_PSE |
270	    CPUID_TSC |
271	    CPUID_MSR |
272	    CPUID_PAE |
273	    /* CPUID_MCE excluded */
274	    CPUID_CX8 |
275	    CPUID_APIC |
276	    CPUID_SEP |
277	    /* CPUID_MTRR excluded */
278	    CPUID_PGE |
279	    /* CPUID_MCA excluded */
280	    CPUID_CMOV |
281	    CPUID_PAT |
282	    CPUID_PSE36 |
283	    /* CPUID_PSN excluded */
284	    CPUID_CLFSH |
285	    /* CPUID_DS excluded */
286	    /* CPUID_ACPI excluded */
287	    CPUID_MMX |
288	    CPUID_FXSR |
289	    CPUID_SSE |
290	    CPUID_SSE2 |
291	    CPUID_SS |
292	    CPUID_HTT |
293	    /* CPUID_TM excluded */
294	    CPUID_PBE
295};
296
297const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000007 = {
298	.eax = ~0,
299	.ebx =
300	    CPUID_SEF_FSGSBASE |
301	    /* CPUID_SEF_TSC_ADJUST excluded */
302	    /* CPUID_SEF_SGX excluded */
303	    CPUID_SEF_BMI1 |
304	    /* CPUID_SEF_HLE excluded */
305	    /* CPUID_SEF_AVX2 excluded */
306	    CPUID_SEF_FDPEXONLY |
307	    CPUID_SEF_SMEP |
308	    CPUID_SEF_BMI2 |
309	    CPUID_SEF_ERMS |
310	    /* CPUID_SEF_INVPCID excluded, but re-included in VMX */
311	    /* CPUID_SEF_RTM excluded */
312	    /* CPUID_SEF_QM excluded */
313	    CPUID_SEF_FPUCSDS |
314	    /* CPUID_SEF_MPX excluded */
315	    CPUID_SEF_PQE |
316	    /* CPUID_SEF_AVX512F excluded */
317	    /* CPUID_SEF_AVX512DQ excluded */
318	    CPUID_SEF_RDSEED |
319	    CPUID_SEF_ADX |
320	    CPUID_SEF_SMAP |
321	    /* CPUID_SEF_AVX512_IFMA excluded */
322	    CPUID_SEF_CLFLUSHOPT |
323	    CPUID_SEF_CLWB,
324	    /* CPUID_SEF_PT excluded */
325	    /* CPUID_SEF_AVX512PF excluded */
326	    /* CPUID_SEF_AVX512ER excluded */
327	    /* CPUID_SEF_AVX512CD excluded */
328	    /* CPUID_SEF_SHA excluded */
329	    /* CPUID_SEF_AVX512BW excluded */
330	    /* CPUID_SEF_AVX512VL excluded */
331	.ecx =
332	    CPUID_SEF_PREFETCHWT1 |
333	    /* CPUID_SEF_AVX512_VBMI excluded */
334	    CPUID_SEF_UMIP |
335	    /* CPUID_SEF_PKU excluded */
336	    /* CPUID_SEF_OSPKE excluded */
337	    /* CPUID_SEF_WAITPKG excluded */
338	    /* CPUID_SEF_AVX512_VBMI2 excluded */
339	    /* CPUID_SEF_CET_SS excluded */
340	    CPUID_SEF_GFNI |
341	    CPUID_SEF_VAES |
342	    CPUID_SEF_VPCLMULQDQ |
343	    /* CPUID_SEF_AVX512_VNNI excluded */
344	    /* CPUID_SEF_AVX512_BITALG excluded */
345	    /* CPUID_SEF_AVX512_VPOPCNTDQ excluded */
346	    /* CPUID_SEF_MAWAU excluded */
347	    /* CPUID_SEF_RDPID excluded */
348	    CPUID_SEF_CLDEMOTE |
349	    CPUID_SEF_MOVDIRI |
350	    CPUID_SEF_MOVDIR64B,
351	    /* CPUID_SEF_SGXLC excluded */
352	    /* CPUID_SEF_PKS excluded */
353	.edx =
354	    /* CPUID_SEF_AVX512_4VNNIW excluded */
355	    /* CPUID_SEF_AVX512_4FMAPS excluded */
356	    CPUID_SEF_FSRM |
357	    /* CPUID_SEF_AVX512_VP2INTERSECT excluded */
358	    /* CPUID_SEF_SRBDS_CTRL excluded */
359	    CPUID_SEF_MD_CLEAR |
360	    /* CPUID_SEF_TSX_FORCE_ABORT excluded */
361	    CPUID_SEF_SERIALIZE |
362	    /* CPUID_SEF_HYBRID excluded */
363	    /* CPUID_SEF_TSXLDTRK excluded */
364	    /* CPUID_SEF_CET_IBT excluded */
365	    /* CPUID_SEF_IBRS excluded */
366	    /* CPUID_SEF_STIBP excluded */
367	    /* CPUID_SEF_L1D_FLUSH excluded */
368	    CPUID_SEF_ARCH_CAP
369	    /* CPUID_SEF_CORE_CAP excluded */
370	    /* CPUID_SEF_SSBD excluded */
371};
372
373const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000001 = {
374	.eax = ~0,
375	.ebx = ~0,
376	.ecx =
377	    CPUID_LAHF |
378	    CPUID_CMPLEGACY |
379	    /* CPUID_SVM excluded */
380	    /* CPUID_EAPIC excluded */
381	    CPUID_ALTMOVCR0 |
382	    CPUID_ABM |
383	    CPUID_SSE4A |
384	    CPUID_MISALIGNSSE |
385	    CPUID_3DNOWPF |
386	    /* CPUID_OSVW excluded */
387	    /* CPUID_IBS excluded */
388	    CPUID_XOP |
389	    /* CPUID_SKINIT excluded */
390	    /* CPUID_WDT excluded */
391	    /* CPUID_LWP excluded */
392	    CPUID_FMA4 |
393	    CPUID_TCE |
394	    /* CPUID_NODEID excluded */
395	    CPUID_TBM |
396	    CPUID_TOPOEXT,
397	    /* CPUID_PCEC excluded */
398	    /* CPUID_PCENB excluded */
399	    /* CPUID_SPM excluded */
400	    /* CPUID_DBE excluded */
401	    /* CPUID_PTSC excluded */
402	    /* CPUID_L2IPERFC excluded */
403	    /* CPUID_MWAITX excluded */
404	.edx =
405	    CPUID_SYSCALL |
406	    CPUID_MPC |
407	    CPUID_XD |
408	    CPUID_MMXX |
409	    CPUID_MMX |
410	    CPUID_FXSR |
411	    CPUID_FFXSR |
412	    CPUID_PAGE1GB |
413	    /* CPUID_RDTSCP excluded */
414	    CPUID_EM64T |
415	    CPUID_3DNOW2 |
416	    CPUID_3DNOW
417};
418
419const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007 = {
420	.eax = 0,
421	.ebx = 0,
422	.ecx = 0,
423	.edx = CPUID_APM_ITSC
424};
425
426const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008 = {
427	.eax = ~0,
428	.ebx =
429	    CPUID_CAPEX_CLZERO |
430	    /* CPUID_CAPEX_IRPERF excluded */
431	    CPUID_CAPEX_XSAVEERPTR |
432	    /* CPUID_CAPEX_RDPRU excluded */
433	    /* CPUID_CAPEX_MCOMMIT excluded */
434	    CPUID_CAPEX_WBNOINVD,
435	.ecx = ~0, /* TODO? */
436	.edx = 0
437};
438
439bool
440nvmm_x86_pat_validate(uint64_t val)
441{
442	uint8_t *pat = (uint8_t *)&val;
443	size_t i;
444
445	for (i = 0; i < 8; i++) {
446		if (__predict_false(pat[i] & ~__BITS(2,0)))
447			return false;
448		if (__predict_false(pat[i] == 2 || pat[i] == 3))
449			return false;
450	}
451
452	return true;
453}
454