1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * tools/testing/selftests/kvm/lib/x86_64/processor.c 4 * 5 * Copyright (C) 2018, Google LLC. 6 */ 7 8#include "linux/bitmap.h" 9#include "test_util.h" 10#include "kvm_util.h" 11#include "processor.h" 12#include "sev.h" 13 14#ifndef NUM_INTERRUPTS 15#define NUM_INTERRUPTS 256 16#endif 17 18#define DEFAULT_CODE_SELECTOR 0x8 19#define DEFAULT_DATA_SELECTOR 0x10 20 21#define MAX_NR_CPUID_ENTRIES 100 22 23vm_vaddr_t exception_handlers; 24bool host_cpu_is_amd; 25bool host_cpu_is_intel; 26 27static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent) 28{ 29 fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx " 30 "rcx: 0x%.16llx rdx: 0x%.16llx\n", 31 indent, "", 32 regs->rax, regs->rbx, regs->rcx, regs->rdx); 33 fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx " 34 "rsp: 0x%.16llx rbp: 0x%.16llx\n", 35 indent, "", 36 regs->rsi, regs->rdi, regs->rsp, regs->rbp); 37 fprintf(stream, "%*sr8: 0x%.16llx r9: 0x%.16llx " 38 "r10: 0x%.16llx r11: 0x%.16llx\n", 39 indent, "", 40 regs->r8, regs->r9, regs->r10, regs->r11); 41 fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx " 42 "r14: 0x%.16llx r15: 0x%.16llx\n", 43 indent, "", 44 regs->r12, regs->r13, regs->r14, regs->r15); 45 fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n", 46 indent, "", 47 regs->rip, regs->rflags); 48} 49 50static void segment_dump(FILE *stream, struct kvm_segment *segment, 51 uint8_t indent) 52{ 53 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x " 54 "selector: 0x%.4x type: 0x%.2x\n", 55 indent, "", segment->base, segment->limit, 56 segment->selector, segment->type); 57 fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x " 58 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n", 59 indent, "", segment->present, segment->dpl, 60 segment->db, segment->s, segment->l); 61 fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x " 62 "unusable: 0x%.2x padding: 0x%.2x\n", 63 indent, "", segment->g, segment->avl, 64 segment->unusable, segment->padding); 65} 66 67static void dtable_dump(FILE *stream, struct kvm_dtable *dtable, 68 uint8_t indent) 69{ 70 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x " 71 "padding: 0x%.4x 0x%.4x 0x%.4x\n", 72 indent, "", dtable->base, dtable->limit, 73 dtable->padding[0], dtable->padding[1], dtable->padding[2]); 74} 75 76static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent) 77{ 78 unsigned int i; 79 80 fprintf(stream, "%*scs:\n", indent, ""); 81 segment_dump(stream, &sregs->cs, indent + 2); 82 fprintf(stream, "%*sds:\n", indent, ""); 83 segment_dump(stream, &sregs->ds, indent + 2); 84 fprintf(stream, "%*ses:\n", indent, ""); 85 segment_dump(stream, &sregs->es, indent + 2); 86 fprintf(stream, "%*sfs:\n", indent, ""); 87 segment_dump(stream, &sregs->fs, indent + 2); 88 fprintf(stream, "%*sgs:\n", indent, ""); 89 segment_dump(stream, &sregs->gs, indent + 2); 90 fprintf(stream, "%*sss:\n", indent, ""); 91 segment_dump(stream, &sregs->ss, indent + 2); 92 fprintf(stream, "%*str:\n", indent, ""); 93 segment_dump(stream, &sregs->tr, indent + 2); 94 fprintf(stream, "%*sldt:\n", indent, ""); 95 segment_dump(stream, &sregs->ldt, indent + 2); 96 97 fprintf(stream, "%*sgdt:\n", indent, ""); 98 dtable_dump(stream, &sregs->gdt, indent + 2); 99 fprintf(stream, "%*sidt:\n", indent, ""); 100 dtable_dump(stream, &sregs->idt, indent + 2); 101 102 fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx " 103 "cr3: 0x%.16llx cr4: 0x%.16llx\n", 104 indent, "", 105 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4); 106 fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx " 107 "apic_base: 0x%.16llx\n", 108 indent, "", 109 sregs->cr8, sregs->efer, sregs->apic_base); 110 111 fprintf(stream, "%*sinterrupt_bitmap:\n", indent, ""); 112 for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) { 113 fprintf(stream, "%*s%.16llx\n", indent + 2, "", 114 sregs->interrupt_bitmap[i]); 115 } 116} 117 118bool kvm_is_tdp_enabled(void) 119{ 120 if (host_cpu_is_intel) 121 return get_kvm_intel_param_bool("ept"); 122 else 123 return get_kvm_amd_param_bool("npt"); 124} 125 126void virt_arch_pgd_alloc(struct kvm_vm *vm) 127{ 128 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " 129 "unknown or unsupported guest mode, mode: 0x%x", vm->mode); 130 131 /* If needed, create page map l4 table. */ 132 if (!vm->pgd_created) { 133 vm->pgd = vm_alloc_page_table(vm); 134 vm->pgd_created = true; 135 } 136} 137 138static void *virt_get_pte(struct kvm_vm *vm, uint64_t *parent_pte, 139 uint64_t vaddr, int level) 140{ 141 uint64_t pt_gpa = PTE_GET_PA(*parent_pte); 142 uint64_t *page_table = addr_gpa2hva(vm, pt_gpa); 143 int index = (vaddr >> PG_LEVEL_SHIFT(level)) & 0x1ffu; 144 145 TEST_ASSERT((*parent_pte & PTE_PRESENT_MASK) || parent_pte == &vm->pgd, 146 "Parent PTE (level %d) not PRESENT for gva: 0x%08lx", 147 level + 1, vaddr); 148 149 return &page_table[index]; 150} 151 152static uint64_t *virt_create_upper_pte(struct kvm_vm *vm, 153 uint64_t *parent_pte, 154 uint64_t vaddr, 155 uint64_t paddr, 156 int current_level, 157 int target_level) 158{ 159 uint64_t *pte = virt_get_pte(vm, parent_pte, vaddr, current_level); 160 161 paddr = vm_untag_gpa(vm, paddr); 162 163 if (!(*pte & PTE_PRESENT_MASK)) { 164 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK; 165 if (current_level == target_level) 166 *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK); 167 else 168 *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK; 169 } else { 170 /* 171 * Entry already present. Assert that the caller doesn't want 172 * a hugepage at this level, and that there isn't a hugepage at 173 * this level. 174 */ 175 TEST_ASSERT(current_level != target_level, 176 "Cannot create hugepage at level: %u, vaddr: 0x%lx", 177 current_level, vaddr); 178 TEST_ASSERT(!(*pte & PTE_LARGE_MASK), 179 "Cannot create page table at level: %u, vaddr: 0x%lx", 180 current_level, vaddr); 181 } 182 return pte; 183} 184 185void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level) 186{ 187 const uint64_t pg_size = PG_LEVEL_SIZE(level); 188 uint64_t *pml4e, *pdpe, *pde; 189 uint64_t *pte; 190 191 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, 192 "Unknown or unsupported guest mode, mode: 0x%x", vm->mode); 193 194 TEST_ASSERT((vaddr % pg_size) == 0, 195 "Virtual address not aligned,\n" 196 "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size); 197 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)), 198 "Invalid virtual address, vaddr: 0x%lx", vaddr); 199 TEST_ASSERT((paddr % pg_size) == 0, 200 "Physical address not aligned,\n" 201 " paddr: 0x%lx page size: 0x%lx", paddr, pg_size); 202 TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, 203 "Physical address beyond maximum supported,\n" 204 " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", 205 paddr, vm->max_gfn, vm->page_size); 206 TEST_ASSERT(vm_untag_gpa(vm, paddr) == paddr, 207 "Unexpected bits in paddr: %lx", paddr); 208 209 /* 210 * Allocate upper level page tables, if not already present. Return 211 * early if a hugepage was created. 212 */ 213 pml4e = virt_create_upper_pte(vm, &vm->pgd, vaddr, paddr, PG_LEVEL_512G, level); 214 if (*pml4e & PTE_LARGE_MASK) 215 return; 216 217 pdpe = virt_create_upper_pte(vm, pml4e, vaddr, paddr, PG_LEVEL_1G, level); 218 if (*pdpe & PTE_LARGE_MASK) 219 return; 220 221 pde = virt_create_upper_pte(vm, pdpe, vaddr, paddr, PG_LEVEL_2M, level); 222 if (*pde & PTE_LARGE_MASK) 223 return; 224 225 /* Fill in page table entry. */ 226 pte = virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K); 227 TEST_ASSERT(!(*pte & PTE_PRESENT_MASK), 228 "PTE already present for 4k page at vaddr: 0x%lx", vaddr); 229 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK); 230 231 /* 232 * Neither SEV nor TDX supports shared page tables, so only the final 233 * leaf PTE needs manually set the C/S-bit. 234 */ 235 if (vm_is_gpa_protected(vm, paddr)) 236 *pte |= vm->arch.c_bit; 237 else 238 *pte |= vm->arch.s_bit; 239} 240 241void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 242{ 243 __virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K); 244} 245 246void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 247 uint64_t nr_bytes, int level) 248{ 249 uint64_t pg_size = PG_LEVEL_SIZE(level); 250 uint64_t nr_pages = nr_bytes / pg_size; 251 int i; 252 253 TEST_ASSERT(nr_bytes % pg_size == 0, 254 "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx", 255 nr_bytes, pg_size); 256 257 for (i = 0; i < nr_pages; i++) { 258 __virt_pg_map(vm, vaddr, paddr, level); 259 260 vaddr += pg_size; 261 paddr += pg_size; 262 } 263} 264 265static bool vm_is_target_pte(uint64_t *pte, int *level, int current_level) 266{ 267 if (*pte & PTE_LARGE_MASK) { 268 TEST_ASSERT(*level == PG_LEVEL_NONE || 269 *level == current_level, 270 "Unexpected hugepage at level %d", current_level); 271 *level = current_level; 272 } 273 274 return *level == current_level; 275} 276 277uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr, 278 int *level) 279{ 280 uint64_t *pml4e, *pdpe, *pde; 281 282 TEST_ASSERT(!vm->arch.is_pt_protected, 283 "Walking page tables of protected guests is impossible"); 284 285 TEST_ASSERT(*level >= PG_LEVEL_NONE && *level < PG_LEVEL_NUM, 286 "Invalid PG_LEVEL_* '%d'", *level); 287 288 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " 289 "unknown or unsupported guest mode, mode: 0x%x", vm->mode); 290 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, 291 (vaddr >> vm->page_shift)), 292 "Invalid virtual address, vaddr: 0x%lx", 293 vaddr); 294 /* 295 * Based on the mode check above there are 48 bits in the vaddr, so 296 * shift 16 to sign extend the last bit (bit-47), 297 */ 298 TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16), 299 "Canonical check failed. The virtual address is invalid."); 300 301 pml4e = virt_get_pte(vm, &vm->pgd, vaddr, PG_LEVEL_512G); 302 if (vm_is_target_pte(pml4e, level, PG_LEVEL_512G)) 303 return pml4e; 304 305 pdpe = virt_get_pte(vm, pml4e, vaddr, PG_LEVEL_1G); 306 if (vm_is_target_pte(pdpe, level, PG_LEVEL_1G)) 307 return pdpe; 308 309 pde = virt_get_pte(vm, pdpe, vaddr, PG_LEVEL_2M); 310 if (vm_is_target_pte(pde, level, PG_LEVEL_2M)) 311 return pde; 312 313 return virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K); 314} 315 316uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr) 317{ 318 int level = PG_LEVEL_4K; 319 320 return __vm_get_page_table_entry(vm, vaddr, &level); 321} 322 323void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 324{ 325 uint64_t *pml4e, *pml4e_start; 326 uint64_t *pdpe, *pdpe_start; 327 uint64_t *pde, *pde_start; 328 uint64_t *pte, *pte_start; 329 330 if (!vm->pgd_created) 331 return; 332 333 fprintf(stream, "%*s " 334 " no\n", indent, ""); 335 fprintf(stream, "%*s index hvaddr gpaddr " 336 "addr w exec dirty\n", 337 indent, ""); 338 pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd); 339 for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) { 340 pml4e = &pml4e_start[n1]; 341 if (!(*pml4e & PTE_PRESENT_MASK)) 342 continue; 343 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u " 344 " %u\n", 345 indent, "", 346 pml4e - pml4e_start, pml4e, 347 addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e), 348 !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK)); 349 350 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK); 351 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) { 352 pdpe = &pdpe_start[n2]; 353 if (!(*pdpe & PTE_PRESENT_MASK)) 354 continue; 355 fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10llx " 356 "%u %u\n", 357 indent, "", 358 pdpe - pdpe_start, pdpe, 359 addr_hva2gpa(vm, pdpe), 360 PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK), 361 !!(*pdpe & PTE_NX_MASK)); 362 363 pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK); 364 for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) { 365 pde = &pde_start[n3]; 366 if (!(*pde & PTE_PRESENT_MASK)) 367 continue; 368 fprintf(stream, "%*spde 0x%-3zx %p " 369 "0x%-12lx 0x%-10llx %u %u\n", 370 indent, "", pde - pde_start, pde, 371 addr_hva2gpa(vm, pde), 372 PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK), 373 !!(*pde & PTE_NX_MASK)); 374 375 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK); 376 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) { 377 pte = &pte_start[n4]; 378 if (!(*pte & PTE_PRESENT_MASK)) 379 continue; 380 fprintf(stream, "%*spte 0x%-3zx %p " 381 "0x%-12lx 0x%-10llx %u %u " 382 " %u 0x%-10lx\n", 383 indent, "", 384 pte - pte_start, pte, 385 addr_hva2gpa(vm, pte), 386 PTE_GET_PFN(*pte), 387 !!(*pte & PTE_WRITABLE_MASK), 388 !!(*pte & PTE_NX_MASK), 389 !!(*pte & PTE_DIRTY_MASK), 390 ((uint64_t) n1 << 27) 391 | ((uint64_t) n2 << 18) 392 | ((uint64_t) n3 << 9) 393 | ((uint64_t) n4)); 394 } 395 } 396 } 397 } 398} 399 400/* 401 * Set Unusable Segment 402 * 403 * Input Args: None 404 * 405 * Output Args: 406 * segp - Pointer to segment register 407 * 408 * Return: None 409 * 410 * Sets the segment register pointed to by @segp to an unusable state. 411 */ 412static void kvm_seg_set_unusable(struct kvm_segment *segp) 413{ 414 memset(segp, 0, sizeof(*segp)); 415 segp->unusable = true; 416} 417 418static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp) 419{ 420 void *gdt = addr_gva2hva(vm, vm->gdt); 421 struct desc64 *desc = gdt + (segp->selector >> 3) * 8; 422 423 desc->limit0 = segp->limit & 0xFFFF; 424 desc->base0 = segp->base & 0xFFFF; 425 desc->base1 = segp->base >> 16; 426 desc->type = segp->type; 427 desc->s = segp->s; 428 desc->dpl = segp->dpl; 429 desc->p = segp->present; 430 desc->limit1 = segp->limit >> 16; 431 desc->avl = segp->avl; 432 desc->l = segp->l; 433 desc->db = segp->db; 434 desc->g = segp->g; 435 desc->base2 = segp->base >> 24; 436 if (!segp->s) 437 desc->base3 = segp->base >> 32; 438} 439 440 441/* 442 * Set Long Mode Flat Kernel Code Segment 443 * 444 * Input Args: 445 * vm - VM whose GDT is being filled, or NULL to only write segp 446 * selector - selector value 447 * 448 * Output Args: 449 * segp - Pointer to KVM segment 450 * 451 * Return: None 452 * 453 * Sets up the KVM segment pointed to by @segp, to be a code segment 454 * with the selector value given by @selector. 455 */ 456static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector, 457 struct kvm_segment *segp) 458{ 459 memset(segp, 0, sizeof(*segp)); 460 segp->selector = selector; 461 segp->limit = 0xFFFFFFFFu; 462 segp->s = 0x1; /* kTypeCodeData */ 463 segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed 464 * | kFlagCodeReadable 465 */ 466 segp->g = true; 467 segp->l = true; 468 segp->present = 1; 469 if (vm) 470 kvm_seg_fill_gdt_64bit(vm, segp); 471} 472 473/* 474 * Set Long Mode Flat Kernel Data Segment 475 * 476 * Input Args: 477 * vm - VM whose GDT is being filled, or NULL to only write segp 478 * selector - selector value 479 * 480 * Output Args: 481 * segp - Pointer to KVM segment 482 * 483 * Return: None 484 * 485 * Sets up the KVM segment pointed to by @segp, to be a data segment 486 * with the selector value given by @selector. 487 */ 488static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector, 489 struct kvm_segment *segp) 490{ 491 memset(segp, 0, sizeof(*segp)); 492 segp->selector = selector; 493 segp->limit = 0xFFFFFFFFu; 494 segp->s = 0x1; /* kTypeCodeData */ 495 segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed 496 * | kFlagDataWritable 497 */ 498 segp->g = true; 499 segp->present = true; 500 if (vm) 501 kvm_seg_fill_gdt_64bit(vm, segp); 502} 503 504vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 505{ 506 int level = PG_LEVEL_NONE; 507 uint64_t *pte = __vm_get_page_table_entry(vm, gva, &level); 508 509 TEST_ASSERT(*pte & PTE_PRESENT_MASK, 510 "Leaf PTE not PRESENT for gva: 0x%08lx", gva); 511 512 /* 513 * No need for a hugepage mask on the PTE, x86-64 requires the "unused" 514 * address bits to be zero. 515 */ 516 return vm_untag_gpa(vm, PTE_GET_PA(*pte)) | (gva & ~HUGEPAGE_MASK(level)); 517} 518 519static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt) 520{ 521 if (!vm->gdt) 522 vm->gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 523 524 dt->base = vm->gdt; 525 dt->limit = getpagesize(); 526} 527 528static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp, 529 int selector) 530{ 531 if (!vm->tss) 532 vm->tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 533 534 memset(segp, 0, sizeof(*segp)); 535 segp->base = vm->tss; 536 segp->limit = 0x67; 537 segp->selector = selector; 538 segp->type = 0xb; 539 segp->present = 1; 540 kvm_seg_fill_gdt_64bit(vm, segp); 541} 542 543static void vcpu_setup(struct kvm_vm *vm, struct kvm_vcpu *vcpu) 544{ 545 struct kvm_sregs sregs; 546 547 /* Set mode specific system register values. */ 548 vcpu_sregs_get(vcpu, &sregs); 549 550 sregs.idt.limit = 0; 551 552 kvm_setup_gdt(vm, &sregs.gdt); 553 554 switch (vm->mode) { 555 case VM_MODE_PXXV48_4K: 556 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG; 557 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR; 558 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX); 559 560 kvm_seg_set_unusable(&sregs.ldt); 561 kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs); 562 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds); 563 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es); 564 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18); 565 break; 566 567 default: 568 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); 569 } 570 571 sregs.cr3 = vm->pgd; 572 vcpu_sregs_set(vcpu, &sregs); 573} 574 575void kvm_arch_vm_post_create(struct kvm_vm *vm) 576{ 577 vm_create_irqchip(vm); 578 sync_global_to_guest(vm, host_cpu_is_intel); 579 sync_global_to_guest(vm, host_cpu_is_amd); 580 581 if (vm->subtype == VM_SUBTYPE_SEV) 582 sev_vm_init(vm); 583 else if (vm->subtype == VM_SUBTYPE_SEV_ES) 584 sev_es_vm_init(vm); 585} 586 587void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) 588{ 589 struct kvm_regs regs; 590 591 vcpu_regs_get(vcpu, ®s); 592 regs.rip = (unsigned long) guest_code; 593 vcpu_regs_set(vcpu, ®s); 594} 595 596struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id) 597{ 598 struct kvm_mp_state mp_state; 599 struct kvm_regs regs; 600 vm_vaddr_t stack_vaddr; 601 struct kvm_vcpu *vcpu; 602 603 stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(), 604 DEFAULT_GUEST_STACK_VADDR_MIN, 605 MEM_REGION_DATA); 606 607 stack_vaddr += DEFAULT_STACK_PGS * getpagesize(); 608 609 /* 610 * Align stack to match calling sequence requirements in section "The 611 * Stack Frame" of the System V ABI AMD64 Architecture Processor 612 * Supplement, which requires the value (%rsp + 8) to be a multiple of 613 * 16 when control is transferred to the function entry point. 614 * 615 * If this code is ever used to launch a vCPU with 32-bit entry point it 616 * may need to subtract 4 bytes instead of 8 bytes. 617 */ 618 TEST_ASSERT(IS_ALIGNED(stack_vaddr, PAGE_SIZE), 619 "__vm_vaddr_alloc() did not provide a page-aligned address"); 620 stack_vaddr -= 8; 621 622 vcpu = __vm_vcpu_add(vm, vcpu_id); 623 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid()); 624 vcpu_setup(vm, vcpu); 625 626 /* Setup guest general purpose registers */ 627 vcpu_regs_get(vcpu, ®s); 628 regs.rflags = regs.rflags | 0x2; 629 regs.rsp = stack_vaddr; 630 vcpu_regs_set(vcpu, ®s); 631 632 /* Setup the MP state */ 633 mp_state.mp_state = 0; 634 vcpu_mp_state_set(vcpu, &mp_state); 635 636 return vcpu; 637} 638 639struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id) 640{ 641 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id); 642 643 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid()); 644 645 return vcpu; 646} 647 648void vcpu_arch_free(struct kvm_vcpu *vcpu) 649{ 650 if (vcpu->cpuid) 651 free(vcpu->cpuid); 652} 653 654/* Do not use kvm_supported_cpuid directly except for validity checks. */ 655static void *kvm_supported_cpuid; 656 657const struct kvm_cpuid2 *kvm_get_supported_cpuid(void) 658{ 659 int kvm_fd; 660 661 if (kvm_supported_cpuid) 662 return kvm_supported_cpuid; 663 664 kvm_supported_cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES); 665 kvm_fd = open_kvm_dev_path_or_exit(); 666 667 kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, 668 (struct kvm_cpuid2 *)kvm_supported_cpuid); 669 670 close(kvm_fd); 671 return kvm_supported_cpuid; 672} 673 674static uint32_t __kvm_cpu_has(const struct kvm_cpuid2 *cpuid, 675 uint32_t function, uint32_t index, 676 uint8_t reg, uint8_t lo, uint8_t hi) 677{ 678 const struct kvm_cpuid_entry2 *entry; 679 int i; 680 681 for (i = 0; i < cpuid->nent; i++) { 682 entry = &cpuid->entries[i]; 683 684 /* 685 * The output registers in kvm_cpuid_entry2 are in alphabetical 686 * order, but kvm_x86_cpu_feature matches that mess, so yay 687 * pointer shenanigans! 688 */ 689 if (entry->function == function && entry->index == index) 690 return ((&entry->eax)[reg] & GENMASK(hi, lo)) >> lo; 691 } 692 693 return 0; 694} 695 696bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid, 697 struct kvm_x86_cpu_feature feature) 698{ 699 return __kvm_cpu_has(cpuid, feature.function, feature.index, 700 feature.reg, feature.bit, feature.bit); 701} 702 703uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid, 704 struct kvm_x86_cpu_property property) 705{ 706 return __kvm_cpu_has(cpuid, property.function, property.index, 707 property.reg, property.lo_bit, property.hi_bit); 708} 709 710uint64_t kvm_get_feature_msr(uint64_t msr_index) 711{ 712 struct { 713 struct kvm_msrs header; 714 struct kvm_msr_entry entry; 715 } buffer = {}; 716 int r, kvm_fd; 717 718 buffer.header.nmsrs = 1; 719 buffer.entry.index = msr_index; 720 kvm_fd = open_kvm_dev_path_or_exit(); 721 722 r = __kvm_ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header); 723 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_GET_MSRS, r)); 724 725 close(kvm_fd); 726 return buffer.entry.data; 727} 728 729void __vm_xsave_require_permission(uint64_t xfeature, const char *name) 730{ 731 int kvm_fd; 732 u64 bitmask; 733 long rc; 734 struct kvm_device_attr attr = { 735 .group = 0, 736 .attr = KVM_X86_XCOMP_GUEST_SUPP, 737 .addr = (unsigned long) &bitmask, 738 }; 739 740 TEST_ASSERT(!kvm_supported_cpuid, 741 "kvm_get_supported_cpuid() cannot be used before ARCH_REQ_XCOMP_GUEST_PERM"); 742 743 TEST_ASSERT(is_power_of_2(xfeature), 744 "Dynamic XFeatures must be enabled one at a time"); 745 746 kvm_fd = open_kvm_dev_path_or_exit(); 747 rc = __kvm_ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr); 748 close(kvm_fd); 749 750 if (rc == -1 && (errno == ENXIO || errno == EINVAL)) 751 __TEST_REQUIRE(0, "KVM_X86_XCOMP_GUEST_SUPP not supported"); 752 753 TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc); 754 755 __TEST_REQUIRE(bitmask & xfeature, 756 "Required XSAVE feature '%s' not supported", name); 757 758 TEST_REQUIRE(!syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, ilog2(xfeature))); 759 760 rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask); 761 TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc); 762 TEST_ASSERT(bitmask & xfeature, 763 "'%s' (0x%lx) not permitted after prctl(ARCH_REQ_XCOMP_GUEST_PERM) permitted=0x%lx", 764 name, xfeature, bitmask); 765} 766 767void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid) 768{ 769 TEST_ASSERT(cpuid != vcpu->cpuid, "@cpuid can't be the vCPU's CPUID"); 770 771 /* Allow overriding the default CPUID. */ 772 if (vcpu->cpuid && vcpu->cpuid->nent < cpuid->nent) { 773 free(vcpu->cpuid); 774 vcpu->cpuid = NULL; 775 } 776 777 if (!vcpu->cpuid) 778 vcpu->cpuid = allocate_kvm_cpuid2(cpuid->nent); 779 780 memcpy(vcpu->cpuid, cpuid, kvm_cpuid2_size(cpuid->nent)); 781 vcpu_set_cpuid(vcpu); 782} 783 784void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, 785 struct kvm_x86_cpu_property property, 786 uint32_t value) 787{ 788 struct kvm_cpuid_entry2 *entry; 789 790 entry = __vcpu_get_cpuid_entry(vcpu, property.function, property.index); 791 792 (&entry->eax)[property.reg] &= ~GENMASK(property.hi_bit, property.lo_bit); 793 (&entry->eax)[property.reg] |= value << property.lo_bit; 794 795 vcpu_set_cpuid(vcpu); 796 797 /* Sanity check that @value doesn't exceed the bounds in any way. */ 798 TEST_ASSERT_EQ(kvm_cpuid_property(vcpu->cpuid, property), value); 799} 800 801void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function) 802{ 803 struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function); 804 805 entry->eax = 0; 806 entry->ebx = 0; 807 entry->ecx = 0; 808 entry->edx = 0; 809 vcpu_set_cpuid(vcpu); 810} 811 812void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu, 813 struct kvm_x86_cpu_feature feature, 814 bool set) 815{ 816 struct kvm_cpuid_entry2 *entry; 817 u32 *reg; 818 819 entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index); 820 reg = (&entry->eax) + feature.reg; 821 822 if (set) 823 *reg |= BIT(feature.bit); 824 else 825 *reg &= ~BIT(feature.bit); 826 827 vcpu_set_cpuid(vcpu); 828} 829 830uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index) 831{ 832 struct { 833 struct kvm_msrs header; 834 struct kvm_msr_entry entry; 835 } buffer = {}; 836 837 buffer.header.nmsrs = 1; 838 buffer.entry.index = msr_index; 839 840 vcpu_msrs_get(vcpu, &buffer.header); 841 842 return buffer.entry.data; 843} 844 845int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value) 846{ 847 struct { 848 struct kvm_msrs header; 849 struct kvm_msr_entry entry; 850 } buffer = {}; 851 852 memset(&buffer, 0, sizeof(buffer)); 853 buffer.header.nmsrs = 1; 854 buffer.entry.index = msr_index; 855 buffer.entry.data = msr_value; 856 857 return __vcpu_ioctl(vcpu, KVM_SET_MSRS, &buffer.header); 858} 859 860void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) 861{ 862 va_list ap; 863 struct kvm_regs regs; 864 865 TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n" 866 " num: %u", 867 num); 868 869 va_start(ap, num); 870 vcpu_regs_get(vcpu, ®s); 871 872 if (num >= 1) 873 regs.rdi = va_arg(ap, uint64_t); 874 875 if (num >= 2) 876 regs.rsi = va_arg(ap, uint64_t); 877 878 if (num >= 3) 879 regs.rdx = va_arg(ap, uint64_t); 880 881 if (num >= 4) 882 regs.rcx = va_arg(ap, uint64_t); 883 884 if (num >= 5) 885 regs.r8 = va_arg(ap, uint64_t); 886 887 if (num >= 6) 888 regs.r9 = va_arg(ap, uint64_t); 889 890 vcpu_regs_set(vcpu, ®s); 891 va_end(ap); 892} 893 894void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) 895{ 896 struct kvm_regs regs; 897 struct kvm_sregs sregs; 898 899 fprintf(stream, "%*svCPU ID: %u\n", indent, "", vcpu->id); 900 901 fprintf(stream, "%*sregs:\n", indent + 2, ""); 902 vcpu_regs_get(vcpu, ®s); 903 regs_dump(stream, ®s, indent + 4); 904 905 fprintf(stream, "%*ssregs:\n", indent + 2, ""); 906 vcpu_sregs_get(vcpu, &sregs); 907 sregs_dump(stream, &sregs, indent + 4); 908} 909 910static struct kvm_msr_list *__kvm_get_msr_index_list(bool feature_msrs) 911{ 912 struct kvm_msr_list *list; 913 struct kvm_msr_list nmsrs; 914 int kvm_fd, r; 915 916 kvm_fd = open_kvm_dev_path_or_exit(); 917 918 nmsrs.nmsrs = 0; 919 if (!feature_msrs) 920 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs); 921 else 922 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, &nmsrs); 923 924 TEST_ASSERT(r == -1 && errno == E2BIG, 925 "Expected -E2BIG, got rc: %i errno: %i (%s)", 926 r, errno, strerror(errno)); 927 928 list = malloc(sizeof(*list) + nmsrs.nmsrs * sizeof(list->indices[0])); 929 TEST_ASSERT(list, "-ENOMEM when allocating MSR index list"); 930 list->nmsrs = nmsrs.nmsrs; 931 932 if (!feature_msrs) 933 kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list); 934 else 935 kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, list); 936 close(kvm_fd); 937 938 TEST_ASSERT(list->nmsrs == nmsrs.nmsrs, 939 "Number of MSRs in list changed, was %d, now %d", 940 nmsrs.nmsrs, list->nmsrs); 941 return list; 942} 943 944const struct kvm_msr_list *kvm_get_msr_index_list(void) 945{ 946 static const struct kvm_msr_list *list; 947 948 if (!list) 949 list = __kvm_get_msr_index_list(false); 950 return list; 951} 952 953 954const struct kvm_msr_list *kvm_get_feature_msr_index_list(void) 955{ 956 static const struct kvm_msr_list *list; 957 958 if (!list) 959 list = __kvm_get_msr_index_list(true); 960 return list; 961} 962 963bool kvm_msr_is_in_save_restore_list(uint32_t msr_index) 964{ 965 const struct kvm_msr_list *list = kvm_get_msr_index_list(); 966 int i; 967 968 for (i = 0; i < list->nmsrs; ++i) { 969 if (list->indices[i] == msr_index) 970 return true; 971 } 972 973 return false; 974} 975 976static void vcpu_save_xsave_state(struct kvm_vcpu *vcpu, 977 struct kvm_x86_state *state) 978{ 979 int size = vm_check_cap(vcpu->vm, KVM_CAP_XSAVE2); 980 981 if (size) { 982 state->xsave = malloc(size); 983 vcpu_xsave2_get(vcpu, state->xsave); 984 } else { 985 state->xsave = malloc(sizeof(struct kvm_xsave)); 986 vcpu_xsave_get(vcpu, state->xsave); 987 } 988} 989 990struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu) 991{ 992 const struct kvm_msr_list *msr_list = kvm_get_msr_index_list(); 993 struct kvm_x86_state *state; 994 int i; 995 996 static int nested_size = -1; 997 998 if (nested_size == -1) { 999 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE); 1000 TEST_ASSERT(nested_size <= sizeof(state->nested_), 1001 "Nested state size too big, %i > %zi", 1002 nested_size, sizeof(state->nested_)); 1003 } 1004 1005 /* 1006 * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees 1007 * guest state is consistent only after userspace re-enters the 1008 * kernel with KVM_RUN. Complete IO prior to migrating state 1009 * to a new VM. 1010 */ 1011 vcpu_run_complete_io(vcpu); 1012 1013 state = malloc(sizeof(*state) + msr_list->nmsrs * sizeof(state->msrs.entries[0])); 1014 TEST_ASSERT(state, "-ENOMEM when allocating kvm state"); 1015 1016 vcpu_events_get(vcpu, &state->events); 1017 vcpu_mp_state_get(vcpu, &state->mp_state); 1018 vcpu_regs_get(vcpu, &state->regs); 1019 vcpu_save_xsave_state(vcpu, state); 1020 1021 if (kvm_has_cap(KVM_CAP_XCRS)) 1022 vcpu_xcrs_get(vcpu, &state->xcrs); 1023 1024 vcpu_sregs_get(vcpu, &state->sregs); 1025 1026 if (nested_size) { 1027 state->nested.size = sizeof(state->nested_); 1028 1029 vcpu_nested_state_get(vcpu, &state->nested); 1030 TEST_ASSERT(state->nested.size <= nested_size, 1031 "Nested state size too big, %i (KVM_CHECK_CAP gave %i)", 1032 state->nested.size, nested_size); 1033 } else { 1034 state->nested.size = 0; 1035 } 1036 1037 state->msrs.nmsrs = msr_list->nmsrs; 1038 for (i = 0; i < msr_list->nmsrs; i++) 1039 state->msrs.entries[i].index = msr_list->indices[i]; 1040 vcpu_msrs_get(vcpu, &state->msrs); 1041 1042 vcpu_debugregs_get(vcpu, &state->debugregs); 1043 1044 return state; 1045} 1046 1047void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state) 1048{ 1049 vcpu_sregs_set(vcpu, &state->sregs); 1050 vcpu_msrs_set(vcpu, &state->msrs); 1051 1052 if (kvm_has_cap(KVM_CAP_XCRS)) 1053 vcpu_xcrs_set(vcpu, &state->xcrs); 1054 1055 vcpu_xsave_set(vcpu, state->xsave); 1056 vcpu_events_set(vcpu, &state->events); 1057 vcpu_mp_state_set(vcpu, &state->mp_state); 1058 vcpu_debugregs_set(vcpu, &state->debugregs); 1059 vcpu_regs_set(vcpu, &state->regs); 1060 1061 if (state->nested.size) 1062 vcpu_nested_state_set(vcpu, &state->nested); 1063} 1064 1065void kvm_x86_state_cleanup(struct kvm_x86_state *state) 1066{ 1067 free(state->xsave); 1068 free(state); 1069} 1070 1071void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits) 1072{ 1073 if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) { 1074 *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32; 1075 *va_bits = 32; 1076 } else { 1077 *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR); 1078 *va_bits = kvm_cpu_property(X86_PROPERTY_MAX_VIRT_ADDR); 1079 } 1080} 1081 1082void kvm_init_vm_address_properties(struct kvm_vm *vm) 1083{ 1084 if (vm->subtype == VM_SUBTYPE_SEV || vm->subtype == VM_SUBTYPE_SEV_ES) { 1085 vm->arch.c_bit = BIT_ULL(this_cpu_property(X86_PROPERTY_SEV_C_BIT)); 1086 vm->gpa_tag_mask = vm->arch.c_bit; 1087 } 1088} 1089 1090static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr, 1091 int dpl, unsigned short selector) 1092{ 1093 struct idt_entry *base = 1094 (struct idt_entry *)addr_gva2hva(vm, vm->idt); 1095 struct idt_entry *e = &base[vector]; 1096 1097 memset(e, 0, sizeof(*e)); 1098 e->offset0 = addr; 1099 e->selector = selector; 1100 e->ist = 0; 1101 e->type = 14; 1102 e->dpl = dpl; 1103 e->p = 1; 1104 e->offset1 = addr >> 16; 1105 e->offset2 = addr >> 32; 1106} 1107 1108 1109static bool kvm_fixup_exception(struct ex_regs *regs) 1110{ 1111 if (regs->r9 != KVM_EXCEPTION_MAGIC || regs->rip != regs->r10) 1112 return false; 1113 1114 if (regs->vector == DE_VECTOR) 1115 return false; 1116 1117 regs->rip = regs->r11; 1118 regs->r9 = regs->vector; 1119 regs->r10 = regs->error_code; 1120 return true; 1121} 1122 1123void route_exception(struct ex_regs *regs) 1124{ 1125 typedef void(*handler)(struct ex_regs *); 1126 handler *handlers = (handler *)exception_handlers; 1127 1128 if (handlers && handlers[regs->vector]) { 1129 handlers[regs->vector](regs); 1130 return; 1131 } 1132 1133 if (kvm_fixup_exception(regs)) 1134 return; 1135 1136 ucall_assert(UCALL_UNHANDLED, 1137 "Unhandled exception in guest", __FILE__, __LINE__, 1138 "Unhandled exception '0x%lx' at guest RIP '0x%lx'", 1139 regs->vector, regs->rip); 1140} 1141 1142void vm_init_descriptor_tables(struct kvm_vm *vm) 1143{ 1144 extern void *idt_handlers; 1145 int i; 1146 1147 vm->idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 1148 vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA); 1149 /* Handlers have the same address in both address spaces.*/ 1150 for (i = 0; i < NUM_INTERRUPTS; i++) 1151 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0, 1152 DEFAULT_CODE_SELECTOR); 1153} 1154 1155void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu) 1156{ 1157 struct kvm_vm *vm = vcpu->vm; 1158 struct kvm_sregs sregs; 1159 1160 vcpu_sregs_get(vcpu, &sregs); 1161 sregs.idt.base = vm->idt; 1162 sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1; 1163 sregs.gdt.base = vm->gdt; 1164 sregs.gdt.limit = getpagesize() - 1; 1165 kvm_seg_set_kernel_data_64bit(NULL, DEFAULT_DATA_SELECTOR, &sregs.gs); 1166 vcpu_sregs_set(vcpu, &sregs); 1167 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; 1168} 1169 1170void vm_install_exception_handler(struct kvm_vm *vm, int vector, 1171 void (*handler)(struct ex_regs *)) 1172{ 1173 vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers); 1174 1175 handlers[vector] = (vm_vaddr_t)handler; 1176} 1177 1178void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) 1179{ 1180 struct ucall uc; 1181 1182 if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) 1183 REPORT_GUEST_ASSERT(uc); 1184} 1185 1186const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid, 1187 uint32_t function, uint32_t index) 1188{ 1189 int i; 1190 1191 for (i = 0; i < cpuid->nent; i++) { 1192 if (cpuid->entries[i].function == function && 1193 cpuid->entries[i].index == index) 1194 return &cpuid->entries[i]; 1195 } 1196 1197 TEST_FAIL("CPUID function 0x%x index 0x%x not found ", function, index); 1198 1199 return NULL; 1200} 1201 1202#define X86_HYPERCALL(inputs...) \ 1203({ \ 1204 uint64_t r; \ 1205 \ 1206 asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t" \ 1207 "jnz 1f\n\t" \ 1208 "vmcall\n\t" \ 1209 "jmp 2f\n\t" \ 1210 "1: vmmcall\n\t" \ 1211 "2:" \ 1212 : "=a"(r) \ 1213 : [use_vmmcall] "r" (host_cpu_is_amd), inputs); \ 1214 \ 1215 r; \ 1216}) 1217 1218uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, 1219 uint64_t a3) 1220{ 1221 return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3)); 1222} 1223 1224uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1) 1225{ 1226 return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1)); 1227} 1228 1229void xen_hypercall(uint64_t nr, uint64_t a0, void *a1) 1230{ 1231 GUEST_ASSERT(!__xen_hypercall(nr, a0, a1)); 1232} 1233 1234const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void) 1235{ 1236 static struct kvm_cpuid2 *cpuid; 1237 int kvm_fd; 1238 1239 if (cpuid) 1240 return cpuid; 1241 1242 cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES); 1243 kvm_fd = open_kvm_dev_path_or_exit(); 1244 1245 kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid); 1246 1247 close(kvm_fd); 1248 return cpuid; 1249} 1250 1251void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu) 1252{ 1253 static struct kvm_cpuid2 *cpuid_full; 1254 const struct kvm_cpuid2 *cpuid_sys, *cpuid_hv; 1255 int i, nent = 0; 1256 1257 if (!cpuid_full) { 1258 cpuid_sys = kvm_get_supported_cpuid(); 1259 cpuid_hv = kvm_get_supported_hv_cpuid(); 1260 1261 cpuid_full = allocate_kvm_cpuid2(cpuid_sys->nent + cpuid_hv->nent); 1262 if (!cpuid_full) { 1263 perror("malloc"); 1264 abort(); 1265 } 1266 1267 /* Need to skip KVM CPUID leaves 0x400000xx */ 1268 for (i = 0; i < cpuid_sys->nent; i++) { 1269 if (cpuid_sys->entries[i].function >= 0x40000000 && 1270 cpuid_sys->entries[i].function < 0x40000100) 1271 continue; 1272 cpuid_full->entries[nent] = cpuid_sys->entries[i]; 1273 nent++; 1274 } 1275 1276 memcpy(&cpuid_full->entries[nent], cpuid_hv->entries, 1277 cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2)); 1278 cpuid_full->nent = nent + cpuid_hv->nent; 1279 } 1280 1281 vcpu_init_cpuid(vcpu, cpuid_full); 1282} 1283 1284const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu) 1285{ 1286 struct kvm_cpuid2 *cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES); 1287 1288 vcpu_ioctl(vcpu, KVM_GET_SUPPORTED_HV_CPUID, cpuid); 1289 1290 return cpuid; 1291} 1292 1293unsigned long vm_compute_max_gfn(struct kvm_vm *vm) 1294{ 1295 const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */ 1296 unsigned long ht_gfn, max_gfn, max_pfn; 1297 uint8_t maxphyaddr; 1298 1299 max_gfn = (1ULL << (vm->pa_bits - vm->page_shift)) - 1; 1300 1301 /* Avoid reserved HyperTransport region on AMD processors. */ 1302 if (!host_cpu_is_amd) 1303 return max_gfn; 1304 1305 /* On parts with <40 physical address bits, the area is fully hidden */ 1306 if (vm->pa_bits < 40) 1307 return max_gfn; 1308 1309 /* Before family 17h, the HyperTransport area is just below 1T. */ 1310 ht_gfn = (1 << 28) - num_ht_pages; 1311 if (this_cpu_family() < 0x17) 1312 goto done; 1313 1314 /* 1315 * Otherwise it's at the top of the physical address space, possibly 1316 * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX. Use 1317 * the old conservative value if MAXPHYADDR is not enumerated. 1318 */ 1319 if (!this_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) 1320 goto done; 1321 1322 maxphyaddr = this_cpu_property(X86_PROPERTY_MAX_PHY_ADDR); 1323 max_pfn = (1ULL << (maxphyaddr - vm->page_shift)) - 1; 1324 1325 if (this_cpu_has_p(X86_PROPERTY_PHYS_ADDR_REDUCTION)) 1326 max_pfn >>= this_cpu_property(X86_PROPERTY_PHYS_ADDR_REDUCTION); 1327 1328 ht_gfn = max_pfn - num_ht_pages; 1329done: 1330 return min(max_gfn, ht_gfn - 1); 1331} 1332 1333/* Returns true if kvm_intel was loaded with unrestricted_guest=1. */ 1334bool vm_is_unrestricted_guest(struct kvm_vm *vm) 1335{ 1336 /* Ensure that a KVM vendor-specific module is loaded. */ 1337 if (vm == NULL) 1338 close(open_kvm_dev_path_or_exit()); 1339 1340 return get_kvm_intel_param_bool("unrestricted_guest"); 1341} 1342 1343void kvm_selftest_arch_init(void) 1344{ 1345 host_cpu_is_intel = this_cpu_is_intel(); 1346 host_cpu_is_amd = this_cpu_is_amd(); 1347} 1348 1349bool sys_clocksource_is_based_on_tsc(void) 1350{ 1351 char *clk_name = sys_get_cur_clocksource(); 1352 bool ret = !strcmp(clk_name, "tsc\n") || 1353 !strcmp(clk_name, "hyperv_clocksource_tsc_page\n"); 1354 1355 free(clk_name); 1356 1357 return ret; 1358} 1359