17a338472SThomas Gleixner// SPDX-License-Identifier: GPL-2.0-only
2783e9e51SPaolo Bonzini/*
3cc68765dSAndrew Jones * tools/testing/selftests/kvm/lib/x86_64/processor.c
4783e9e51SPaolo Bonzini *
5783e9e51SPaolo Bonzini * Copyright (C) 2018, Google LLC.
6783e9e51SPaolo Bonzini */
7783e9e51SPaolo Bonzini
8783e9e51SPaolo Bonzini#include "test_util.h"
9783e9e51SPaolo Bonzini#include "kvm_util.h"
10cc68765dSAndrew Jones#include "../kvm_util_internal.h"
11cc68765dSAndrew Jones#include "processor.h"
12783e9e51SPaolo Bonzini
1329faeb96SAaron Lewis#ifndef NUM_INTERRUPTS
1429faeb96SAaron Lewis#define NUM_INTERRUPTS 256
1529faeb96SAaron Lewis#endif
1629faeb96SAaron Lewis
1729faeb96SAaron Lewis#define DEFAULT_CODE_SELECTOR 0x8
1829faeb96SAaron Lewis#define DEFAULT_DATA_SELECTOR 0x10
1929faeb96SAaron Lewis
2029faeb96SAaron Lewisvm_vaddr_t exception_handlers;
2129faeb96SAaron Lewis
22783e9e51SPaolo Bonzini/* Virtual translation table structure declarations */
23b007e904SSean Christophersonstruct pageUpperEntry {
24783e9e51SPaolo Bonzini	uint64_t present:1;
25783e9e51SPaolo Bonzini	uint64_t writable:1;
26783e9e51SPaolo Bonzini	uint64_t user:1;
27783e9e51SPaolo Bonzini	uint64_t write_through:1;
28783e9e51SPaolo Bonzini	uint64_t cache_disable:1;
29783e9e51SPaolo Bonzini	uint64_t accessed:1;
30783e9e51SPaolo Bonzini	uint64_t ignored_06:1;
31783e9e51SPaolo Bonzini	uint64_t page_size:1;
32783e9e51SPaolo Bonzini	uint64_t ignored_11_08:4;
336d96ca6aSSean Christopherson	uint64_t pfn:40;
34783e9e51SPaolo Bonzini	uint64_t ignored_62_52:11;
35783e9e51SPaolo Bonzini	uint64_t execute_disable:1;
36783e9e51SPaolo Bonzini};
37783e9e51SPaolo Bonzini
38783e9e51SPaolo Bonzinistruct pageTableEntry {
39783e9e51SPaolo Bonzini	uint64_t present:1;
40783e9e51SPaolo Bonzini	uint64_t writable:1;
41783e9e51SPaolo Bonzini	uint64_t user:1;
42783e9e51SPaolo Bonzini	uint64_t write_through:1;
43783e9e51SPaolo Bonzini	uint64_t cache_disable:1;
44783e9e51SPaolo Bonzini	uint64_t accessed:1;
45783e9e51SPaolo Bonzini	uint64_t dirty:1;
46783e9e51SPaolo Bonzini	uint64_t reserved_07:1;
47783e9e51SPaolo Bonzini	uint64_t global:1;
48783e9e51SPaolo Bonzini	uint64_t ignored_11_09:3;
496d96ca6aSSean Christopherson	uint64_t pfn:40;
50783e9e51SPaolo Bonzini	uint64_t ignored_62_52:11;
51783e9e51SPaolo Bonzini	uint64_t execute_disable:1;
52783e9e51SPaolo Bonzini};
53783e9e51SPaolo Bonzini
54783e9e51SPaolo Bonzinivoid regs_dump(FILE *stream, struct kvm_regs *regs,
55783e9e51SPaolo Bonzini	       uint8_t indent)
56783e9e51SPaolo Bonzini{
57783e9e51SPaolo Bonzini	fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
58783e9e51SPaolo Bonzini		"rcx: 0x%.16llx rdx: 0x%.16llx\n",
59783e9e51SPaolo Bonzini		indent, "",
60783e9e51SPaolo Bonzini		regs->rax, regs->rbx, regs->rcx, regs->rdx);
61783e9e51SPaolo Bonzini	fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
62783e9e51SPaolo Bonzini		"rsp: 0x%.16llx rbp: 0x%.16llx\n",
63783e9e51SPaolo Bonzini		indent, "",
64783e9e51SPaolo Bonzini		regs->rsi, regs->rdi, regs->rsp, regs->rbp);
65783e9e51SPaolo Bonzini	fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
66783e9e51SPaolo Bonzini		"r10: 0x%.16llx r11: 0x%.16llx\n",
67783e9e51SPaolo Bonzini		indent, "",
68783e9e51SPaolo Bonzini		regs->r8, regs->r9, regs->r10, regs->r11);
69783e9e51SPaolo Bonzini	fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
70783e9e51SPaolo Bonzini		"r14: 0x%.16llx r15: 0x%.16llx\n",
71783e9e51SPaolo Bonzini		indent, "",
72783e9e51SPaolo Bonzini		regs->r12, regs->r13, regs->r14, regs->r15);
73783e9e51SPaolo Bonzini	fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
74783e9e51SPaolo Bonzini		indent, "",
75783e9e51SPaolo Bonzini		regs->rip, regs->rflags);
76783e9e51SPaolo Bonzini}
77783e9e51SPaolo Bonzini
7842593624SAndrew Jones/*
7942593624SAndrew Jones * Segment Dump
80783e9e51SPaolo Bonzini *
81783e9e51SPaolo Bonzini * Input Args:
8242593624SAndrew Jones *   stream  - Output FILE stream
83783e9e51SPaolo Bonzini *   segment - KVM segment
8442593624SAndrew Jones *   indent  - Left margin indent amount
85783e9e51SPaolo Bonzini *
8642593624SAndrew Jones * Output Args: None
87783e9e51SPaolo Bonzini *
88783e9e51SPaolo Bonzini * Return: None
89783e9e51SPaolo Bonzini *
9042593624SAndrew Jones * Dumps the state of the KVM segment given by @segment, to the FILE stream
9142593624SAndrew Jones * given by @stream.
92783e9e51SPaolo Bonzini */
93783e9e51SPaolo Bonzinistatic void segment_dump(FILE *stream, struct kvm_segment *segment,
94783e9e51SPaolo Bonzini			 uint8_t indent)
95783e9e51SPaolo Bonzini{
96783e9e51SPaolo Bonzini	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
97783e9e51SPaolo Bonzini		"selector: 0x%.4x type: 0x%.2x\n",
98783e9e51SPaolo Bonzini		indent, "", segment->base, segment->limit,
99783e9e51SPaolo Bonzini		segment->selector, segment->type);
100783e9e51SPaolo Bonzini	fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
101783e9e51SPaolo Bonzini		"db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
102783e9e51SPaolo Bonzini		indent, "", segment->present, segment->dpl,
103783e9e51SPaolo Bonzini		segment->db, segment->s, segment->l);
104783e9e51SPaolo Bonzini	fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
105783e9e51SPaolo Bonzini		"unusable: 0x%.2x padding: 0x%.2x\n",
106783e9e51SPaolo Bonzini		indent, "", segment->g, segment->avl,
107783e9e51SPaolo Bonzini		segment->unusable, segment->padding);
108783e9e51SPaolo Bonzini}
109783e9e51SPaolo Bonzini
11042593624SAndrew Jones/*
11142593624SAndrew Jones * dtable Dump
112783e9e51SPaolo Bonzini *
113783e9e51SPaolo Bonzini * Input Args:
11442593624SAndrew Jones *   stream - Output FILE stream
115783e9e51SPaolo Bonzini *   dtable - KVM dtable
11642593624SAndrew Jones *   indent - Left margin indent amount
117783e9e51SPaolo Bonzini *
11842593624SAndrew Jones * Output Args: None
119783e9e51SPaolo Bonzini *
120783e9e51SPaolo Bonzini * Return: None
121783e9e51SPaolo Bonzini *
12242593624SAndrew Jones * Dumps the state of the KVM dtable given by @dtable, to the FILE stream
12342593624SAndrew Jones * given by @stream.
124783e9e51SPaolo Bonzini */
125783e9e51SPaolo Bonzinistatic void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
126783e9e51SPaolo Bonzini			uint8_t indent)
127783e9e51SPaolo Bonzini{
128783e9e51SPaolo Bonzini	fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
129783e9e51SPaolo Bonzini		"padding: 0x%.4x 0x%.4x 0x%.4x\n",
130783e9e51SPaolo Bonzini		indent, "", dtable->base, dtable->limit,
131783e9e51SPaolo Bonzini		dtable->padding[0], dtable->padding[1], dtable->padding[2]);
132783e9e51SPaolo Bonzini}
133783e9e51SPaolo Bonzini
134783e9e51SPaolo Bonzinivoid sregs_dump(FILE *stream, struct kvm_sregs *sregs,
135783e9e51SPaolo Bonzini		uint8_t indent)
136783e9e51SPaolo Bonzini{
137783e9e51SPaolo Bonzini	unsigned int i;
138783e9e51SPaolo Bonzini
139783e9e51SPaolo Bonzini	fprintf(stream, "%*scs:\n", indent, "");
140783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->cs, indent + 2);
141783e9e51SPaolo Bonzini	fprintf(stream, "%*sds:\n", indent, "");
142783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->ds, indent + 2);
143783e9e51SPaolo Bonzini	fprintf(stream, "%*ses:\n", indent, "");
144783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->es, indent + 2);
145783e9e51SPaolo Bonzini	fprintf(stream, "%*sfs:\n", indent, "");
146783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->fs, indent + 2);
147783e9e51SPaolo Bonzini	fprintf(stream, "%*sgs:\n", indent, "");
148783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->gs, indent + 2);
149783e9e51SPaolo Bonzini	fprintf(stream, "%*sss:\n", indent, "");
150783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->ss, indent + 2);
151783e9e51SPaolo Bonzini	fprintf(stream, "%*str:\n", indent, "");
152783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->tr, indent + 2);
153783e9e51SPaolo Bonzini	fprintf(stream, "%*sldt:\n", indent, "");
154783e9e51SPaolo Bonzini	segment_dump(stream, &sregs->ldt, indent + 2);
155783e9e51SPaolo Bonzini
156783e9e51SPaolo Bonzini	fprintf(stream, "%*sgdt:\n", indent, "");
157783e9e51SPaolo Bonzini	dtable_dump(stream, &sregs->gdt, indent + 2);
158783e9e51SPaolo Bonzini	fprintf(stream, "%*sidt:\n", indent, "");
159783e9e51SPaolo Bonzini	dtable_dump(stream, &sregs->idt, indent + 2);
160783e9e51SPaolo Bonzini
161783e9e51SPaolo Bonzini	fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
162783e9e51SPaolo Bonzini		"cr3: 0x%.16llx cr4: 0x%.16llx\n",
163783e9e51SPaolo Bonzini		indent, "",
164783e9e51SPaolo Bonzini		sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
165783e9e51SPaolo Bonzini	fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
166783e9e51SPaolo Bonzini		"apic_base: 0x%.16llx\n",
167783e9e51SPaolo Bonzini		indent, "",
168783e9e51SPaolo Bonzini		sregs->cr8, sregs->efer, sregs->apic_base);
169783e9e51SPaolo Bonzini
170783e9e51SPaolo Bonzini	fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
171783e9e51SPaolo Bonzini	for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
172783e9e51SPaolo Bonzini		fprintf(stream, "%*s%.16llx\n", indent + 2, "",
173783e9e51SPaolo Bonzini			sregs->interrupt_bitmap[i]);
174783e9e51SPaolo Bonzini	}
175783e9e51SPaolo Bonzini}
176783e9e51SPaolo Bonzini
177a75a895eSSean Christophersonvoid virt_pgd_alloc(struct kvm_vm *vm)
178783e9e51SPaolo Bonzini{
179567a9f1eSPeter Xu	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
180783e9e51SPaolo Bonzini		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
181783e9e51SPaolo Bonzini
182783e9e51SPaolo Bonzini	/* If needed, create page map l4 table. */
183783e9e51SPaolo Bonzini	if (!vm->pgd_created) {
184cce0c23dSSean Christopherson		vm->pgd = vm_alloc_page_table(vm);
185783e9e51SPaolo Bonzini		vm->pgd_created = true;
186783e9e51SPaolo Bonzini	}
187783e9e51SPaolo Bonzini}
188783e9e51SPaolo Bonzini
189f681d686SSean Christophersonstatic void *virt_get_pte(struct kvm_vm *vm, uint64_t pt_pfn, uint64_t vaddr,
190f681d686SSean Christopherson			  int level)
191f681d686SSean Christopherson{
192f681d686SSean Christopherson	uint64_t *page_table = addr_gpa2hva(vm, pt_pfn << vm->page_shift);
193f681d686SSean Christopherson	int index = vaddr >> (vm->page_shift + level * 9) & 0x1ffu;
194f681d686SSean Christopherson
195f681d686SSean Christopherson	return &page_table[index];
196f681d686SSean Christopherson}
197f681d686SSean Christopherson
198b007e904SSean Christophersonstatic struct pageUpperEntry *virt_create_upper_pte(struct kvm_vm *vm,
199b007e904SSean Christopherson						    uint64_t pt_pfn,
200b007e904SSean Christopherson						    uint64_t vaddr,
201ad5f16e4SSean Christopherson						    uint64_t paddr,
202ad5f16e4SSean Christopherson						    int level,
203ad5f16e4SSean Christopherson						    enum x86_page_size page_size)
204b007e904SSean Christopherson{
205b007e904SSean Christopherson	struct pageUpperEntry *pte = virt_get_pte(vm, pt_pfn, vaddr, level);
206b007e904SSean Christopherson
207b007e904SSean Christopherson	if (!pte->present) {
208b007e904SSean Christopherson		pte->writable = true;
209b007e904SSean Christopherson		pte->present = true;
210ad5f16e4SSean Christopherson		pte->page_size = (level == page_size);
211ad5f16e4SSean Christopherson		if (pte->page_size)
212ad5f16e4SSean Christopherson			pte->pfn = paddr >> vm->page_shift;
213ad5f16e4SSean Christopherson		else
214ad5f16e4SSean Christopherson			pte->pfn = vm_alloc_page_table(vm) >> vm->page_shift;
215ad5f16e4SSean Christopherson	} else {
216ad5f16e4SSean Christopherson		/*
217ad5f16e4SSean Christopherson		 * Entry already present.  Assert that the caller doesn't want
218ad5f16e4SSean Christopherson		 * a hugepage at this level, and that there isn't a hugepage at
219ad5f16e4SSean Christopherson		 * this level.
220ad5f16e4SSean Christopherson		 */
221ad5f16e4SSean Christopherson		TEST_ASSERT(level != page_size,
222ad5f16e4SSean Christopherson			    "Cannot create hugepage at level: %u, vaddr: 0x%lx\n",
223ad5f16e4SSean Christopherson			    page_size, vaddr);
224ad5f16e4SSean Christopherson		TEST_ASSERT(!pte->page_size,
225ad5f16e4SSean Christopherson			    "Cannot create page table at level: %u, vaddr: 0x%lx\n",
226ad5f16e4SSean Christopherson			    level, vaddr);
227b007e904SSean Christopherson	}
228b007e904SSean Christopherson	return pte;
229b007e904SSean Christopherson}
230b007e904SSean Christopherson
231ad5f16e4SSean Christophersonvoid __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
232ad5f16e4SSean Christopherson		   enum x86_page_size page_size)
233783e9e51SPaolo Bonzini{
234ad5f16e4SSean Christopherson	const uint64_t pg_size = 1ull << ((page_size * 9) + 12);
235b007e904SSean Christopherson	struct pageUpperEntry *pml4e, *pdpe, *pde;
236f681d686SSean Christopherson	struct pageTableEntry *pte;
237783e9e51SPaolo Bonzini
238ad5f16e4SSean Christopherson	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,
239ad5f16e4SSean Christopherson		    "Unknown or unsupported guest mode, mode: 0x%x", vm->mode);
240783e9e51SPaolo Bonzini
241ad5f16e4SSean Christopherson	TEST_ASSERT((vaddr % pg_size) == 0,
242ad5f16e4SSean Christopherson		    "Virtual address not aligned,\n"
243ad5f16e4SSean Christopherson		    "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size);
244ad5f16e4SSean Christopherson	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)),
245ad5f16e4SSean Christopherson		    "Invalid virtual address, vaddr: 0x%lx", vaddr);
246ad5f16e4SSean Christopherson	TEST_ASSERT((paddr % pg_size) == 0,
247ad5f16e4SSean Christopherson		    "Physical address not aligned,\n"
248ad5f16e4SSean Christopherson		    "  paddr: 0x%lx page size: 0x%lx", paddr, pg_size);
249783e9e51SPaolo Bonzini	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
250ad5f16e4SSean Christopherson		    "Physical address beyond maximum supported,\n"
251ad5f16e4SSean Christopherson		    "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
252ad5f16e4SSean Christopherson		    paddr, vm->max_gfn, vm->page_size);
253ad5f16e4SSean Christopherson
254ad5f16e4SSean Christopherson	/*
255ad5f16e4SSean Christopherson	 * Allocate upper level page tables, if not already present.  Return
256ad5f16e4SSean Christopherson	 * early if a hugepage was created.
257ad5f16e4SSean Christopherson	 */
258ad5f16e4SSean Christopherson	pml4e = virt_create_upper_pte(vm, vm->pgd >> vm->page_shift,
259ad5f16e4SSean Christopherson				      vaddr, paddr, 3, page_size);
260ad5f16e4SSean Christopherson	if (pml4e->page_size)
261ad5f16e4SSean Christopherson		return;
262ad5f16e4SSean Christopherson
263ad5f16e4SSean Christopherson	pdpe = virt_create_upper_pte(vm, pml4e->pfn, vaddr, paddr, 2, page_size);
264ad5f16e4SSean Christopherson	if (pdpe->page_size)
265ad5f16e4SSean Christopherson		return;
266783e9e51SPaolo Bonzini
267ad5f16e4SSean Christopherson	pde = virt_create_upper_pte(vm, pdpe->pfn, vaddr, paddr, 1, page_size);
268ad5f16e4SSean Christopherson	if (pde->page_size)
269ad5f16e4SSean Christopherson		return;
270783e9e51SPaolo Bonzini
271783e9e51SPaolo Bonzini	/* Fill in page table entry. */
272f681d686SSean Christopherson	pte = virt_get_pte(vm, pde->pfn, vaddr, 0);
273ad5f16e4SSean Christopherson	TEST_ASSERT(!pte->present,
274ad5f16e4SSean Christopherson		    "PTE already present for 4k page at vaddr: 0x%lx\n", vaddr);
275f681d686SSean Christopherson	pte->pfn = paddr >> vm->page_shift;
276f681d686SSean Christopherson	pte->writable = true;
277f681d686SSean Christopherson	pte->present = 1;
278783e9e51SPaolo Bonzini}
279783e9e51SPaolo Bonzini
280ad5f16e4SSean Christophersonvoid virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
281ad5f16e4SSean Christopherson{
282ad5f16e4SSean Christopherson	__virt_pg_map(vm, vaddr, paddr, X86_PAGE_SIZE_4K);
283ad5f16e4SSean Christopherson}
284ad5f16e4SSean Christopherson
28539bbcc3aSAaron Lewisstatic struct pageTableEntry *_vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid,
28639bbcc3aSAaron Lewis						       uint64_t vaddr)
28739bbcc3aSAaron Lewis{
28839bbcc3aSAaron Lewis	uint16_t index[4];
28939bbcc3aSAaron Lewis	struct pageUpperEntry *pml4e, *pdpe, *pde;
29039bbcc3aSAaron Lewis	struct pageTableEntry *pte;
29139bbcc3aSAaron Lewis	struct kvm_cpuid_entry2 *entry;
29239bbcc3aSAaron Lewis	struct kvm_sregs sregs;
29339bbcc3aSAaron Lewis	int max_phy_addr;
29439bbcc3aSAaron Lewis	/* Set the bottom 52 bits. */
29539bbcc3aSAaron Lewis	uint64_t rsvd_mask = 0x000fffffffffffff;
29639bbcc3aSAaron Lewis
29739bbcc3aSAaron Lewis	entry = kvm_get_supported_cpuid_index(0x80000008, 0);
29839bbcc3aSAaron Lewis	max_phy_addr = entry->eax & 0x000000ff;
29939bbcc3aSAaron Lewis	/* Clear the bottom bits of the reserved mask. */
30039bbcc3aSAaron Lewis	rsvd_mask = (rsvd_mask >> max_phy_addr) << max_phy_addr;
30139bbcc3aSAaron Lewis
30239bbcc3aSAaron Lewis	/*
30339bbcc3aSAaron Lewis	 * SDM vol 3, fig 4-11 "Formats of CR3 and Paging-Structure Entries
30439bbcc3aSAaron Lewis	 * with 4-Level Paging and 5-Level Paging".
30539bbcc3aSAaron Lewis	 * If IA32_EFER.NXE = 0 and the P flag of a paging-structure entry is 1,
30639bbcc3aSAaron Lewis	 * the XD flag (bit 63) is reserved.
30739bbcc3aSAaron Lewis	 */
30839bbcc3aSAaron Lewis	vcpu_sregs_get(vm, vcpuid, &sregs);
30939bbcc3aSAaron Lewis	if ((sregs.efer & EFER_NX) == 0) {
31039bbcc3aSAaron Lewis		rsvd_mask |= (1ull << 63);
31139bbcc3aSAaron Lewis	}
31239bbcc3aSAaron Lewis
31339bbcc3aSAaron Lewis	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
31439bbcc3aSAaron Lewis		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
31539bbcc3aSAaron Lewis	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
31639bbcc3aSAaron Lewis		(vaddr >> vm->page_shift)),
31739bbcc3aSAaron Lewis		"Invalid virtual address, vaddr: 0x%lx",
31839bbcc3aSAaron Lewis		vaddr);
31939bbcc3aSAaron Lewis	/*
32039bbcc3aSAaron Lewis	 * Based on the mode check above there are 48 bits in the vaddr, so
32139bbcc3aSAaron Lewis	 * shift 16 to sign extend the last bit (bit-47),
32239bbcc3aSAaron Lewis	 */
32339bbcc3aSAaron Lewis	TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16),
32439bbcc3aSAaron Lewis		"Canonical check failed.  The virtual address is invalid.");
32539bbcc3aSAaron Lewis
32639bbcc3aSAaron Lewis	index[0] = (vaddr >> 12) & 0x1ffu;
32739bbcc3aSAaron Lewis	index[1] = (vaddr >> 21) & 0x1ffu;
32839bbcc3aSAaron Lewis	index[2] = (vaddr >> 30) & 0x1ffu;
32939bbcc3aSAaron Lewis	index[3] = (vaddr >> 39) & 0x1ffu;
33039bbcc3aSAaron Lewis
33139bbcc3aSAaron Lewis	pml4e = addr_gpa2hva(vm, vm->pgd);
33239bbcc3aSAaron Lewis	TEST_ASSERT(pml4e[index[3]].present,
33339bbcc3aSAaron Lewis		"Expected pml4e to be present for gva: 0x%08lx", vaddr);
33439bbcc3aSAaron Lewis	TEST_ASSERT((*(uint64_t*)(&pml4e[index[3]]) &
33539bbcc3aSAaron Lewis		(rsvd_mask | (1ull << 7))) == 0,
33639bbcc3aSAaron Lewis		"Unexpected reserved bits set.");
33739bbcc3aSAaron Lewis
33839bbcc3aSAaron Lewis	pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size);
33939bbcc3aSAaron Lewis	TEST_ASSERT(pdpe[index[2]].present,
34039bbcc3aSAaron Lewis		"Expected pdpe to be present for gva: 0x%08lx", vaddr);
34139bbcc3aSAaron Lewis	TEST_ASSERT(pdpe[index[2]].page_size == 0,
34239bbcc3aSAaron Lewis		"Expected pdpe to map a pde not a 1-GByte page.");
34339bbcc3aSAaron Lewis	TEST_ASSERT((*(uint64_t*)(&pdpe[index[2]]) & rsvd_mask) == 0,
34439bbcc3aSAaron Lewis		"Unexpected reserved bits set.");
34539bbcc3aSAaron Lewis
34639bbcc3aSAaron Lewis	pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size);
34739bbcc3aSAaron Lewis	TEST_ASSERT(pde[index[1]].present,
34839bbcc3aSAaron Lewis		"Expected pde to be present for gva: 0x%08lx", vaddr);
34939bbcc3aSAaron Lewis	TEST_ASSERT(pde[index[1]].page_size == 0,
35039bbcc3aSAaron Lewis		"Expected pde to map a pte not a 2-MByte page.");
35139bbcc3aSAaron Lewis	TEST_ASSERT((*(uint64_t*)(&pde[index[1]]) & rsvd_mask) == 0,
35239bbcc3aSAaron Lewis		"Unexpected reserved bits set.");
35339bbcc3aSAaron Lewis
35439bbcc3aSAaron Lewis	pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size);
35539bbcc3aSAaron Lewis	TEST_ASSERT(pte[index[0]].present,
35639bbcc3aSAaron Lewis		"Expected pte to be present for gva: 0x%08lx", vaddr);
35739bbcc3aSAaron Lewis
35839bbcc3aSAaron Lewis	return &pte[index[0]];
35939bbcc3aSAaron Lewis}
36039bbcc3aSAaron Lewis
36139bbcc3aSAaron Lewisuint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr)
36239bbcc3aSAaron Lewis{
36339bbcc3aSAaron Lewis	struct pageTableEntry *pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
36439bbcc3aSAaron Lewis
36539bbcc3aSAaron Lewis	return *(uint64_t *)pte;
36639bbcc3aSAaron Lewis}
36739bbcc3aSAaron Lewis
36839bbcc3aSAaron Lewisvoid vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr,
36939bbcc3aSAaron Lewis			     uint64_t pte)
37039bbcc3aSAaron Lewis{
37139bbcc3aSAaron Lewis	struct pageTableEntry *new_pte = _vm_get_page_table_entry(vm, vcpuid,
37239bbcc3aSAaron Lewis								  vaddr);
37339bbcc3aSAaron Lewis
37439bbcc3aSAaron Lewis	*(uint64_t *)new_pte = pte;
37539bbcc3aSAaron Lewis}
37639bbcc3aSAaron Lewis
377783e9e51SPaolo Bonzinivoid virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
378783e9e51SPaolo Bonzini{
379b007e904SSean Christopherson	struct pageUpperEntry *pml4e, *pml4e_start;
380b007e904SSean Christopherson	struct pageUpperEntry *pdpe, *pdpe_start;
381b007e904SSean Christopherson	struct pageUpperEntry *pde, *pde_start;
382783e9e51SPaolo Bonzini	struct pageTableEntry *pte, *pte_start;
383783e9e51SPaolo Bonzini
384783e9e51SPaolo Bonzini	if (!vm->pgd_created)
385783e9e51SPaolo Bonzini		return;
386783e9e51SPaolo Bonzini
387783e9e51SPaolo Bonzini	fprintf(stream, "%*s                                          "
388783e9e51SPaolo Bonzini		"                no\n", indent, "");
389783e9e51SPaolo Bonzini	fprintf(stream, "%*s      index hvaddr         gpaddr         "
390783e9e51SPaolo Bonzini		"addr         w exec dirty\n",
391783e9e51SPaolo Bonzini		indent, "");
392b007e904SSean Christopherson	pml4e_start = (struct pageUpperEntry *) addr_gpa2hva(vm, vm->pgd);
393783e9e51SPaolo Bonzini	for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
394783e9e51SPaolo Bonzini		pml4e = &pml4e_start[n1];
395783e9e51SPaolo Bonzini		if (!pml4e->present)
396783e9e51SPaolo Bonzini			continue;
397783e9e51SPaolo Bonzini		fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10lx %u "
398783e9e51SPaolo Bonzini			" %u\n",
399783e9e51SPaolo Bonzini			indent, "",
400783e9e51SPaolo Bonzini			pml4e - pml4e_start, pml4e,
4016d96ca6aSSean Christopherson			addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->pfn,
402783e9e51SPaolo Bonzini			pml4e->writable, pml4e->execute_disable);
403783e9e51SPaolo Bonzini
4046d96ca6aSSean Christopherson		pdpe_start = addr_gpa2hva(vm, pml4e->pfn * vm->page_size);
405783e9e51SPaolo Bonzini		for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
406783e9e51SPaolo Bonzini			pdpe = &pdpe_start[n2];
407783e9e51SPaolo Bonzini			if (!pdpe->present)
408783e9e51SPaolo Bonzini				continue;
409783e9e51SPaolo Bonzini			fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10lx "
410783e9e51SPaolo Bonzini				"%u  %u\n",
411783e9e51SPaolo Bonzini				indent, "",
412783e9e51SPaolo Bonzini				pdpe - pdpe_start, pdpe,
413783e9e51SPaolo Bonzini				addr_hva2gpa(vm, pdpe),
4146d96ca6aSSean Christopherson				(uint64_t) pdpe->pfn, pdpe->writable,
415783e9e51SPaolo Bonzini				pdpe->execute_disable);
416783e9e51SPaolo Bonzini
4176d96ca6aSSean Christopherson			pde_start = addr_gpa2hva(vm, pdpe->pfn * vm->page_size);
418783e9e51SPaolo Bonzini			for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
419783e9e51SPaolo Bonzini				pde = &pde_start[n3];
420783e9e51SPaolo Bonzini				if (!pde->present)
421783e9e51SPaolo Bonzini					continue;
422783e9e51SPaolo Bonzini				fprintf(stream, "%*spde   0x%-3zx %p "
423783e9e51SPaolo Bonzini					"0x%-12lx 0x%-10lx %u  %u\n",
424783e9e51SPaolo Bonzini					indent, "", pde - pde_start, pde,
425783e9e51SPaolo Bonzini					addr_hva2gpa(vm, pde),
4266d96ca6aSSean Christopherson					(uint64_t) pde->pfn, pde->writable,
427783e9e51SPaolo Bonzini					pde->execute_disable);
428783e9e51SPaolo Bonzini
4296d96ca6aSSean Christopherson				pte_start = addr_gpa2hva(vm, pde->pfn * vm->page_size);
430783e9e51SPaolo Bonzini				for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
431783e9e51SPaolo Bonzini					pte = &pte_start[n4];
432783e9e51SPaolo Bonzini					if (!pte->present)
433783e9e51SPaolo Bonzini						continue;
434783e9e51SPaolo Bonzini					fprintf(stream, "%*spte   0x%-3zx %p "
435783e9e51SPaolo Bonzini						"0x%-12lx 0x%-10lx %u  %u "
436783e9e51SPaolo Bonzini						"    %u    0x%-10lx\n",
437783e9e51SPaolo Bonzini						indent, "",
438783e9e51SPaolo Bonzini						pte - pte_start, pte,
439783e9e51SPaolo Bonzini						addr_hva2gpa(vm, pte),
4406d96ca6aSSean Christopherson						(uint64_t) pte->pfn,
441783e9e51SPaolo Bonzini						pte->writable,
442783e9e51SPaolo Bonzini						pte->execute_disable,
443783e9e51SPaolo Bonzini						pte->dirty,
444783e9e51SPaolo Bonzini						((uint64_t) n1 << 27)
445783e9e51SPaolo Bonzini							| ((uint64_t) n2 << 18)
446783e9e51SPaolo Bonzini							| ((uint64_t) n3 << 9)
447783e9e51SPaolo Bonzini							| ((uint64_t) n4));
448783e9e51SPaolo Bonzini				}
449783e9e51SPaolo Bonzini			}
450783e9e51SPaolo Bonzini		}
451783e9e51SPaolo Bonzini	}
452783e9e51SPaolo Bonzini}
453783e9e51SPaolo Bonzini
45442593624SAndrew Jones/*
45542593624SAndrew Jones * Set Unusable Segment
456783e9e51SPaolo Bonzini *
457783e9e51SPaolo Bonzini * Input Args: None
458783e9e51SPaolo Bonzini *
459783e9e51SPaolo Bonzini * Output Args:
460783e9e51SPaolo Bonzini *   segp - Pointer to segment register
461783e9e51SPaolo Bonzini *
462783e9e51SPaolo Bonzini * Return: None
463783e9e51SPaolo Bonzini *
46442593624SAndrew Jones * Sets the segment register pointed to by @segp to an unusable state.
465783e9e51SPaolo Bonzini */
466783e9e51SPaolo Bonzinistatic void kvm_seg_set_unusable(struct kvm_segment *segp)
467783e9e51SPaolo Bonzini{
468783e9e51SPaolo Bonzini	memset(segp, 0, sizeof(*segp));
469783e9e51SPaolo Bonzini	segp->unusable = true;
470783e9e51SPaolo Bonzini}
471783e9e51SPaolo Bonzini
4722305339eSPaolo Bonzinistatic void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
4732305339eSPaolo Bonzini{
4742305339eSPaolo Bonzini	void *gdt = addr_gva2hva(vm, vm->gdt);
4752305339eSPaolo Bonzini	struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
4762305339eSPaolo Bonzini
4772305339eSPaolo Bonzini	desc->limit0 = segp->limit & 0xFFFF;
4782305339eSPaolo Bonzini	desc->base0 = segp->base & 0xFFFF;
4792305339eSPaolo Bonzini	desc->base1 = segp->base >> 16;
4802305339eSPaolo Bonzini	desc->type = segp->type;
481df11f7ddSAaron Lewis	desc->s = segp->s;
4822305339eSPaolo Bonzini	desc->dpl = segp->dpl;
4832305339eSPaolo Bonzini	desc->p = segp->present;
4842305339eSPaolo Bonzini	desc->limit1 = segp->limit >> 16;
485df11f7ddSAaron Lewis	desc->avl = segp->avl;
4862305339eSPaolo Bonzini	desc->l = segp->l;
4872305339eSPaolo Bonzini	desc->db = segp->db;
4882305339eSPaolo Bonzini	desc->g = segp->g;
4892305339eSPaolo Bonzini	desc->base2 = segp->base >> 24;
4902305339eSPaolo Bonzini	if (!segp->s)
4912305339eSPaolo Bonzini		desc->base3 = segp->base >> 32;
4922305339eSPaolo Bonzini}
4932305339eSPaolo Bonzini
4942305339eSPaolo Bonzini
49542593624SAndrew Jones/*
49642593624SAndrew Jones * Set Long Mode Flat Kernel Code Segment
497783e9e51SPaolo Bonzini *
498783e9e51SPaolo Bonzini * Input Args:
4992305339eSPaolo Bonzini *   vm - VM whose GDT is being filled, or NULL to only write segp
500783e9e51SPaolo Bonzini *   selector - selector value
501783e9e51SPaolo Bonzini *
502783e9e51SPaolo Bonzini * Output Args:
503783e9e51SPaolo Bonzini *   segp - Pointer to KVM segment
504783e9e51SPaolo Bonzini *
505783e9e51SPaolo Bonzini * Return: None
506783e9e51SPaolo Bonzini *
50742593624SAndrew Jones * Sets up the KVM segment pointed to by @segp, to be a code segment
50842593624SAndrew Jones * with the selector value given by @selector.
509783e9e51SPaolo Bonzini */
5102305339eSPaolo Bonzinistatic void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
511783e9e51SPaolo Bonzini	struct kvm_segment *segp)
512783e9e51SPaolo Bonzini{
513783e9e51SPaolo Bonzini	memset(segp, 0, sizeof(*segp));
514783e9e51SPaolo Bonzini	segp->selector = selector;
515783e9e51SPaolo Bonzini	segp->limit = 0xFFFFFFFFu;
516783e9e51SPaolo Bonzini	segp->s = 0x1; /* kTypeCodeData */
517783e9e51SPaolo Bonzini	segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
518783e9e51SPaolo Bonzini					  * | kFlagCodeReadable
519783e9e51SPaolo Bonzini					  */
520783e9e51SPaolo Bonzini	segp->g = true;
521783e9e51SPaolo Bonzini	segp->l = true;
522783e9e51SPaolo Bonzini	segp->present = 1;
5232305339eSPaolo Bonzini	if (vm)
5242305339eSPaolo Bonzini		kvm_seg_fill_gdt_64bit(vm, segp);
525783e9e51SPaolo Bonzini}
526783e9e51SPaolo Bonzini
52742593624SAndrew Jones/*
52842593624SAndrew Jones * Set Long Mode Flat Kernel Data Segment
529783e9e51SPaolo Bonzini *
530783e9e51SPaolo Bonzini * Input Args:
5312305339eSPaolo Bonzini *   vm - VM whose GDT is being filled, or NULL to only write segp
532783e9e51SPaolo Bonzini *   selector - selector value
533783e9e51SPaolo Bonzini *
534783e9e51SPaolo Bonzini * Output Args:
535783e9e51SPaolo Bonzini *   segp - Pointer to KVM segment
536783e9e51SPaolo Bonzini *
537783e9e51SPaolo Bonzini * Return: None
538783e9e51SPaolo Bonzini *
53942593624SAndrew Jones * Sets up the KVM segment pointed to by @segp, to be a data segment
54042593624SAndrew Jones * with the selector value given by @selector.
541783e9e51SPaolo Bonzini */
5422305339eSPaolo Bonzinistatic void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
543783e9e51SPaolo Bonzini	struct kvm_segment *segp)
544783e9e51SPaolo Bonzini{
545783e9e51SPaolo Bonzini	memset(segp, 0, sizeof(*segp));
546783e9e51SPaolo Bonzini	segp->selector = selector;
547783e9e51SPaolo Bonzini	segp->limit = 0xFFFFFFFFu;
548783e9e51SPaolo Bonzini	segp->s = 0x1; /* kTypeCodeData */
549783e9e51SPaolo Bonzini	segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
550783e9e51SPaolo Bonzini					  * | kFlagDataWritable
551783e9e51SPaolo Bonzini					  */
552783e9e51SPaolo Bonzini	segp->g = true;
553783e9e51SPaolo Bonzini	segp->present = true;
5542305339eSPaolo Bonzini	if (vm)
5552305339eSPaolo Bonzini		kvm_seg_fill_gdt_64bit(vm, segp);
556783e9e51SPaolo Bonzini}
557783e9e51SPaolo Bonzini
558783e9e51SPaolo Bonzinivm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
559783e9e51SPaolo Bonzini{
560783e9e51SPaolo Bonzini	uint16_t index[4];
561b007e904SSean Christopherson	struct pageUpperEntry *pml4e, *pdpe, *pde;
562783e9e51SPaolo Bonzini	struct pageTableEntry *pte;
563783e9e51SPaolo Bonzini
564567a9f1eSPeter Xu	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
565783e9e51SPaolo Bonzini		"unknown or unsupported guest mode, mode: 0x%x", vm->mode);
566783e9e51SPaolo Bonzini
567783e9e51SPaolo Bonzini	index[0] = (gva >> 12) & 0x1ffu;
568783e9e51SPaolo Bonzini	index[1] = (gva >> 21) & 0x1ffu;
569783e9e51SPaolo Bonzini	index[2] = (gva >> 30) & 0x1ffu;
570783e9e51SPaolo Bonzini	index[3] = (gva >> 39) & 0x1ffu;
571783e9e51SPaolo Bonzini
572783e9e51SPaolo Bonzini	if (!vm->pgd_created)
573783e9e51SPaolo Bonzini		goto unmapped_gva;
574783e9e51SPaolo Bonzini	pml4e = addr_gpa2hva(vm, vm->pgd);
575783e9e51SPaolo Bonzini	if (!pml4e[index[3]].present)
576783e9e51SPaolo Bonzini		goto unmapped_gva;
577783e9e51SPaolo Bonzini
5786d96ca6aSSean Christopherson	pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size);
579783e9e51SPaolo Bonzini	if (!pdpe[index[2]].present)
580783e9e51SPaolo Bonzini		goto unmapped_gva;
581783e9e51SPaolo Bonzini
5826d96ca6aSSean Christopherson	pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size);
583783e9e51SPaolo Bonzini	if (!pde[index[1]].present)
584783e9e51SPaolo Bonzini		goto unmapped_gva;
585783e9e51SPaolo Bonzini
5866d96ca6aSSean Christopherson	pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size);
587783e9e51SPaolo Bonzini	if (!pte[index[0]].present)
588783e9e51SPaolo Bonzini		goto unmapped_gva;
589783e9e51SPaolo Bonzini
5906d96ca6aSSean Christopherson	return (pte[index[0]].pfn * vm->page_size) + (gva & 0xfffu);
591783e9e51SPaolo Bonzini
592783e9e51SPaolo Bonziniunmapped_gva:
593352be2c5SWainer dos Santos Moschetta	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
594319f6f97SThomas Huth	exit(EXIT_FAILURE);
595783e9e51SPaolo Bonzini}
596783e9e51SPaolo Bonzini
5971dcd1c58SSean Christophersonstatic void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
5982305339eSPaolo Bonzini{
5992305339eSPaolo Bonzini	if (!vm->gdt)
6005ae4d870SSean Christopherson		vm->gdt = vm_vaddr_alloc_page(vm);
6012305339eSPaolo Bonzini
6022305339eSPaolo Bonzini	dt->base = vm->gdt;
6032305339eSPaolo Bonzini	dt->limit = getpagesize();
6042305339eSPaolo Bonzini}
6052305339eSPaolo Bonzini
6062305339eSPaolo Bonzinistatic void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
6071dcd1c58SSean Christopherson				int selector)
6082305339eSPaolo Bonzini{
6092305339eSPaolo Bonzini	if (!vm->tss)
6105ae4d870SSean Christopherson		vm->tss = vm_vaddr_alloc_page(vm);
6112305339eSPaolo Bonzini
6122305339eSPaolo Bonzini	memset(segp, 0, sizeof(*segp));
6132305339eSPaolo Bonzini	segp->base = vm->tss;
6142305339eSPaolo Bonzini	segp->limit = 0x67;
6152305339eSPaolo Bonzini	segp->selector = selector;
6162305339eSPaolo Bonzini	segp->type = 0xb;
6172305339eSPaolo Bonzini	segp->present = 1;
6182305339eSPaolo Bonzini	kvm_seg_fill_gdt_64bit(vm, segp);
6192305339eSPaolo Bonzini}
6202305339eSPaolo Bonzini
6211dcd1c58SSean Christophersonstatic void vcpu_setup(struct kvm_vm *vm, int vcpuid)
622783e9e51SPaolo Bonzini{
623783e9e51SPaolo Bonzini	struct kvm_sregs sregs;
624783e9e51SPaolo Bonzini
625783e9e51SPaolo Bonzini	/* Set mode specific system register values. */
626783e9e51SPaolo Bonzini	vcpu_sregs_get(vm, vcpuid, &sregs);
627783e9e51SPaolo Bonzini
6282305339eSPaolo Bonzini	sregs.idt.limit = 0;
6292305339eSPaolo Bonzini
6301dcd1c58SSean Christopherson	kvm_setup_gdt(vm, &sregs.gdt);
6312305339eSPaolo Bonzini
632783e9e51SPaolo Bonzini	switch (vm->mode) {
633567a9f1eSPeter Xu	case VM_MODE_PXXV48_4K:
634783e9e51SPaolo Bonzini		sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
6356c930268SAndrew Jones		sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
636783e9e51SPaolo Bonzini		sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
637783e9e51SPaolo Bonzini
638783e9e51SPaolo Bonzini		kvm_seg_set_unusable(&sregs.ldt);
63929faeb96SAaron Lewis		kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs);
64029faeb96SAaron Lewis		kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds);
64129faeb96SAaron Lewis		kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es);
6421dcd1c58SSean Christopherson		kvm_setup_tss_64bit(vm, &sregs.tr, 0x18);
643783e9e51SPaolo Bonzini		break;
644783e9e51SPaolo Bonzini
645783e9e51SPaolo Bonzini	default:
646352be2c5SWainer dos Santos Moschetta		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
647783e9e51SPaolo Bonzini	}
648783e9e51SPaolo Bonzini
6492305339eSPaolo Bonzini	sregs.cr3 = vm->pgd;
6502305339eSPaolo Bonzini	vcpu_sregs_set(vm, vcpuid, &sregs);
651783e9e51SPaolo Bonzini}
65242593624SAndrew Jones
653415a3c33SWei Wang#define CPUID_XFD_BIT (1 << 4)
654415a3c33SWei Wangstatic bool is_xfd_supported(void)
655415a3c33SWei Wang{
656415a3c33SWei Wang	int eax, ebx, ecx, edx;
657415a3c33SWei Wang	const int leaf = 0xd, subleaf = 0x1;
658415a3c33SWei Wang
659415a3c33SWei Wang	__asm__ __volatile__(
660415a3c33SWei Wang		"cpuid"
661415a3c33SWei Wang		: /* output */ "=a"(eax), "=b"(ebx),
662415a3c33SWei Wang		  "=c"(ecx), "=d"(edx)
663415a3c33SWei Wang		: /* input */ "0"(leaf), "2"(subleaf));
664415a3c33SWei Wang
665415a3c33SWei Wang	return !!(eax & CPUID_XFD_BIT);
666415a3c33SWei Wang}
667415a3c33SWei Wang
668415a3c33SWei Wangvoid vm_xsave_req_perm(void)
669415a3c33SWei Wang{
670415a3c33SWei Wang	unsigned long bitmask;
671415a3c33SWei Wang	long rc;
672415a3c33SWei Wang
673415a3c33SWei Wang	if (!is_xfd_supported())
674415a3c33SWei Wang		return;
675415a3c33SWei Wang
676415a3c33SWei Wang	rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM,
677415a3c33SWei Wang		     XSTATE_XTILE_DATA_BIT);
678415a3c33SWei Wang	/*
679415a3c33SWei Wang	 * The older kernel version(<5.15) can't support
680415a3c33SWei Wang	 * ARCH_REQ_XCOMP_GUEST_PERM and directly return.
681415a3c33SWei Wang	 */
682415a3c33SWei Wang	if (rc)
683415a3c33SWei Wang		return;
684415a3c33SWei Wang
685415a3c33SWei Wang	rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask);
686415a3c33SWei Wang	TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc);
687415a3c33SWei Wang	TEST_ASSERT(bitmask & XFEATURE_XTILE_MASK,
688415a3c33SWei Wang		    "prctl(ARCH_REQ_XCOMP_GUEST_PERM) failure bitmask=0x%lx",
689415a3c33SWei Wang		    bitmask);
690415a3c33SWei Wang}
691415a3c33SWei Wang
692783e9e51SPaolo Bonzinivoid vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
693783e9e51SPaolo Bonzini{
694783e9e51SPaolo Bonzini	struct kvm_mp_state mp_state;
695783e9e51SPaolo Bonzini	struct kvm_regs regs;
696783e9e51SPaolo Bonzini	vm_vaddr_t stack_vaddr;
697783e9e51SPaolo Bonzini	stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
698a75a895eSSean Christopherson				     DEFAULT_GUEST_STACK_VADDR_MIN);
699783e9e51SPaolo Bonzini
700783e9e51SPaolo Bonzini	/* Create VCPU */
701837ec79bSPaolo Bonzini	vm_vcpu_add(vm, vcpuid);
702413eaa4eSMichael Roth	vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid());
7031dcd1c58SSean Christopherson	vcpu_setup(vm, vcpuid);
704783e9e51SPaolo Bonzini
705783e9e51SPaolo Bonzini	/* Setup guest general purpose registers */
706783e9e51SPaolo Bonzini	vcpu_regs_get(vm, vcpuid, &regs);
707783e9e51SPaolo Bonzini	regs.rflags = regs.rflags | 0x2;
708783e9e51SPaolo Bonzini	regs.rsp = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize());
709783e9e51SPaolo Bonzini	regs.rip = (unsigned long) guest_code;
710783e9e51SPaolo Bonzini	vcpu_regs_set(vm, vcpuid, &regs);
711783e9e51SPaolo Bonzini
712783e9e51SPaolo Bonzini	/* Setup the MP state */
713783e9e51SPaolo Bonzini	mp_state.mp_state = 0;
714783e9e51SPaolo Bonzini	vcpu_set_mp_state(vm, vcpuid, &mp_state);
715783e9e51SPaolo Bonzini}
716783e9e51SPaolo Bonzini
71742593624SAndrew Jones/*
71842593624SAndrew Jones * Allocate an instance of struct kvm_cpuid2
719eabe7881SAndrew Jones *
720eabe7881SAndrew Jones * Input Args: None
721eabe7881SAndrew Jones *
722eabe7881SAndrew Jones * Output Args: None
723eabe7881SAndrew Jones *
724eabe7881SAndrew Jones * Return: A pointer to the allocated struct. The caller is responsible
725eabe7881SAndrew Jones * for freeing this struct.
726eabe7881SAndrew Jones *
727eabe7881SAndrew Jones * Since kvm_cpuid2 uses a 0-length array to allow a the size of the
728eabe7881SAndrew Jones * array to be decided at allocation time, allocation is slightly
729eabe7881SAndrew Jones * complicated. This function uses a reasonable default length for
730eabe7881SAndrew Jones * the array and performs the appropriate allocation.
731eabe7881SAndrew Jones */
732eabe7881SAndrew Jonesstatic struct kvm_cpuid2 *allocate_kvm_cpuid2(void)
733eabe7881SAndrew Jones{
734eabe7881SAndrew Jones	struct kvm_cpuid2 *cpuid;
735eabe7881SAndrew Jones	int nent = 100;
736eabe7881SAndrew Jones	size_t size;
737eabe7881SAndrew Jones
738eabe7881SAndrew Jones	size = sizeof(*cpuid);
739eabe7881SAndrew Jones	size += nent * sizeof(struct kvm_cpuid_entry2);
740eabe7881SAndrew Jones	cpuid = malloc(size);
741eabe7881SAndrew Jones	if (!cpuid) {
742eabe7881SAndrew Jones		perror("malloc");
743eabe7881SAndrew Jones		abort();
744eabe7881SAndrew Jones	}
745eabe7881SAndrew Jones
746eabe7881SAndrew Jones	cpuid->nent = nent;
747eabe7881SAndrew Jones
748eabe7881SAndrew Jones	return cpuid;
749eabe7881SAndrew Jones}
750eabe7881SAndrew Jones
75142593624SAndrew Jones/*
75242593624SAndrew Jones * KVM Supported CPUID Get
753eabe7881SAndrew Jones *
754eabe7881SAndrew Jones * Input Args: None
755eabe7881SAndrew Jones *
756eabe7881SAndrew Jones * Output Args:
757eabe7881SAndrew Jones *
758eabe7881SAndrew Jones * Return: The supported KVM CPUID
759eabe7881SAndrew Jones *
760eabe7881SAndrew Jones * Get the guest CPUID supported by KVM.
761eabe7881SAndrew Jones */
762eabe7881SAndrew Jonesstruct kvm_cpuid2 *kvm_get_supported_cpuid(void)
763eabe7881SAndrew Jones{
764eabe7881SAndrew Jones	static struct kvm_cpuid2 *cpuid;
765eabe7881SAndrew Jones	int ret;
766eabe7881SAndrew Jones	int kvm_fd;
767eabe7881SAndrew Jones
768eabe7881SAndrew Jones	if (cpuid)
769eabe7881SAndrew Jones		return cpuid;
770eabe7881SAndrew Jones
771eabe7881SAndrew Jones	cpuid = allocate_kvm_cpuid2();
7722aab4b35SDavid Matlack	kvm_fd = open_kvm_dev_path_or_exit();
773eabe7881SAndrew Jones
774eabe7881SAndrew Jones	ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, cpuid);
775eabe7881SAndrew Jones	TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_CPUID failed %d %d\n",
776eabe7881SAndrew Jones		    ret, errno);
777eabe7881SAndrew Jones
778eabe7881SAndrew Jones	close(kvm_fd);
779eabe7881SAndrew Jones	return cpuid;
780eabe7881SAndrew Jones}
781eabe7881SAndrew Jones
782f88d4f2fSLike Xu/*
783f88d4f2fSLike Xu * KVM Get MSR
784f88d4f2fSLike Xu *
785f88d4f2fSLike Xu * Input Args:
786f88d4f2fSLike Xu *   msr_index - Index of MSR
787f88d4f2fSLike Xu *
788f88d4f2fSLike Xu * Output Args: None
789f88d4f2fSLike Xu *
790f88d4f2fSLike Xu * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
791f88d4f2fSLike Xu *
792f88d4f2fSLike Xu * Get value of MSR for VCPU.
793f88d4f2fSLike Xu */
794f88d4f2fSLike Xuuint64_t kvm_get_feature_msr(uint64_t msr_index)
795f88d4f2fSLike Xu{
796f88d4f2fSLike Xu	struct {
797f88d4f2fSLike Xu		struct kvm_msrs header;
798f88d4f2fSLike Xu		struct kvm_msr_entry entry;
799f88d4f2fSLike Xu	} buffer = {};
800f88d4f2fSLike Xu	int r, kvm_fd;
801f88d4f2fSLike Xu
802f88d4f2fSLike Xu	buffer.header.nmsrs = 1;
803f88d4f2fSLike Xu	buffer.entry.index = msr_index;
8042aab4b35SDavid Matlack	kvm_fd = open_kvm_dev_path_or_exit();
805f88d4f2fSLike Xu
806f88d4f2fSLike Xu	r = ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
807f88d4f2fSLike Xu	TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
808f88d4f2fSLike Xu		"  rc: %i errno: %i", r, errno);
809f88d4f2fSLike Xu
810f88d4f2fSLike Xu	close(kvm_fd);
811f88d4f2fSLike Xu	return buffer.entry.data;
812f88d4f2fSLike Xu}
813f88d4f2fSLike Xu
814fb18d053SVitaly Kuznetsov/*
815fb18d053SVitaly Kuznetsov * VM VCPU CPUID Set
816fb18d053SVitaly Kuznetsov *
817fb18d053SVitaly Kuznetsov * Input Args:
818fb18d053SVitaly Kuznetsov *   vm - Virtual Machine
819fb18d053SVitaly Kuznetsov *   vcpuid - VCPU id
820fb18d053SVitaly Kuznetsov *
821fb18d053SVitaly Kuznetsov * Output Args: None
822fb18d053SVitaly Kuznetsov *
823fb18d053SVitaly Kuznetsov * Return: KVM CPUID (KVM_GET_CPUID2)
824fb18d053SVitaly Kuznetsov *
825fb18d053SVitaly Kuznetsov * Set the VCPU's CPUID.
826fb18d053SVitaly Kuznetsov */
827fb18d053SVitaly Kuznetsovstruct kvm_cpuid2 *vcpu_get_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
828fb18d053SVitaly Kuznetsov{
829fb18d053SVitaly Kuznetsov	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
830fb18d053SVitaly Kuznetsov	struct kvm_cpuid2 *cpuid;
831bcd22e14SPaolo Bonzini	int max_ent;
832bcd22e14SPaolo Bonzini	int rc = -1;
833fb18d053SVitaly Kuznetsov
834fb18d053SVitaly Kuznetsov	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
835fb18d053SVitaly Kuznetsov
836fb18d053SVitaly Kuznetsov	cpuid = allocate_kvm_cpuid2();
837fb18d053SVitaly Kuznetsov	max_ent = cpuid->nent;
838fb18d053SVitaly Kuznetsov
839fb18d053SVitaly Kuznetsov	for (cpuid->nent = 1; cpuid->nent <= max_ent; cpuid->nent++) {
840fb18d053SVitaly Kuznetsov		rc = ioctl(vcpu->fd, KVM_GET_CPUID2, cpuid);
841fb18d053SVitaly Kuznetsov		if (!rc)
842fb18d053SVitaly Kuznetsov			break;
843fb18d053SVitaly Kuznetsov
844fb18d053SVitaly Kuznetsov		TEST_ASSERT(rc == -1 && errno == E2BIG,
845fb18d053SVitaly Kuznetsov			    "KVM_GET_CPUID2 should either succeed or give E2BIG: %d %d",
846fb18d053SVitaly Kuznetsov			    rc, errno);
847