1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * tools/testing/selftests/kvm/include/kvm_util_base.h 4 * 5 * Copyright (C) 2018, Google LLC. 6 */ 7#ifndef SELFTEST_KVM_UTIL_BASE_H 8#define SELFTEST_KVM_UTIL_BASE_H 9 10#include "test_util.h" 11 12#include <linux/compiler.h> 13#include "linux/hashtable.h" 14#include "linux/list.h" 15#include <linux/kernel.h> 16#include <linux/kvm.h> 17#include "linux/rbtree.h" 18#include <linux/types.h> 19 20#include <asm/atomic.h> 21#include <asm/kvm.h> 22 23#include <sys/ioctl.h> 24 25#include "kvm_util_arch.h" 26#include "sparsebit.h" 27 28/* 29 * Provide a version of static_assert() that is guaranteed to have an optional 30 * message param. If _ISOC11_SOURCE is defined, glibc (/usr/include/assert.h) 31 * #undefs and #defines static_assert() as a direct alias to _Static_assert(), 32 * i.e. effectively makes the message mandatory. Many KVM selftests #define 33 * _GNU_SOURCE for various reasons, and _GNU_SOURCE implies _ISOC11_SOURCE. As 34 * a result, static_assert() behavior is non-deterministic and may or may not 35 * require a message depending on #include order. 36 */ 37#define __kvm_static_assert(expr, msg, ...) _Static_assert(expr, msg) 38#define kvm_static_assert(expr, ...) __kvm_static_assert(expr, ##__VA_ARGS__, #expr) 39 40#define KVM_DEV_PATH "/dev/kvm" 41#define KVM_MAX_VCPUS 512 42 43#define NSEC_PER_SEC 1000000000L 44 45typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */ 46typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */ 47 48struct userspace_mem_region { 49 struct kvm_userspace_memory_region2 region; 50 struct sparsebit *unused_phy_pages; 51 struct sparsebit *protected_phy_pages; 52 int fd; 53 off_t offset; 54 enum vm_mem_backing_src_type backing_src_type; 55 void *host_mem; 56 void *host_alias; 57 void *mmap_start; 58 void *mmap_alias; 59 size_t mmap_size; 60 struct rb_node gpa_node; 61 struct rb_node hva_node; 62 struct hlist_node slot_node; 63}; 64 65struct kvm_vcpu { 66 struct list_head list; 67 uint32_t id; 68 int fd; 69 struct kvm_vm *vm; 70 struct kvm_run *run; 71#ifdef __x86_64__ 72 struct kvm_cpuid2 *cpuid; 73#endif 74 struct kvm_dirty_gfn *dirty_gfns; 75 uint32_t fetch_index; 76 uint32_t dirty_gfns_count; 77}; 78 79struct userspace_mem_regions { 80 struct rb_root gpa_tree; 81 struct rb_root hva_tree; 82 DECLARE_HASHTABLE(slot_hash, 9); 83}; 84 85enum kvm_mem_region_type { 86 MEM_REGION_CODE, 87 MEM_REGION_DATA, 88 MEM_REGION_PT, 89 MEM_REGION_TEST_DATA, 90 NR_MEM_REGIONS, 91}; 92 93struct kvm_vm { 94 int mode; 95 unsigned long type; 96 uint8_t subtype; 97 int kvm_fd; 98 int fd; 99 unsigned int pgtable_levels; 100 unsigned int page_size; 101 unsigned int page_shift; 102 unsigned int pa_bits; 103 unsigned int va_bits; 104 uint64_t max_gfn; 105 struct list_head vcpus; 106 struct userspace_mem_regions regions; 107 struct sparsebit *vpages_valid; 108 struct sparsebit *vpages_mapped; 109 bool has_irqchip; 110 bool pgd_created; 111 vm_paddr_t ucall_mmio_addr; 112 vm_paddr_t pgd; 113 vm_vaddr_t gdt; 114 vm_vaddr_t tss; 115 vm_vaddr_t idt; 116 vm_vaddr_t handlers; 117 uint32_t dirty_ring_size; 118 uint64_t gpa_tag_mask; 119 120 struct kvm_vm_arch arch; 121 122 /* Cache of information for binary stats interface */ 123 int stats_fd; 124 struct kvm_stats_header stats_header; 125 struct kvm_stats_desc *stats_desc; 126 127 /* 128 * KVM region slots. These are the default memslots used by page 129 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] 130 * memslot. 131 */ 132 uint32_t memslots[NR_MEM_REGIONS]; 133}; 134 135struct vcpu_reg_sublist { 136 const char *name; 137 long capability; 138 int feature; 139 int feature_type; 140 bool finalize; 141 __u64 *regs; 142 __u64 regs_n; 143 __u64 *rejects_set; 144 __u64 rejects_set_n; 145 __u64 *skips_set; 146 __u64 skips_set_n; 147}; 148 149struct vcpu_reg_list { 150 char *name; 151 struct vcpu_reg_sublist sublists[]; 152}; 153 154#define for_each_sublist(c, s) \ 155 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) 156 157#define kvm_for_each_vcpu(vm, i, vcpu) \ 158 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \ 159 if (!((vcpu) = vm->vcpus[i])) \ 160 continue; \ 161 else 162 163struct userspace_mem_region * 164memslot2region(struct kvm_vm *vm, uint32_t memslot); 165 166static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, 167 enum kvm_mem_region_type type) 168{ 169 assert(type < NR_MEM_REGIONS); 170 return memslot2region(vm, vm->memslots[type]); 171} 172 173/* Minimum allocated guest virtual and physical addresses */ 174#define KVM_UTIL_MIN_VADDR 0x2000 175#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 176 177#define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 178#define DEFAULT_STACK_PGS 5 179 180enum vm_guest_mode { 181 VM_MODE_P52V48_4K, 182 VM_MODE_P52V48_16K, 183 VM_MODE_P52V48_64K, 184 VM_MODE_P48V48_4K, 185 VM_MODE_P48V48_16K, 186 VM_MODE_P48V48_64K, 187 VM_MODE_P40V48_4K, 188 VM_MODE_P40V48_16K, 189 VM_MODE_P40V48_64K, 190 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */ 191 VM_MODE_P47V64_4K, 192 VM_MODE_P44V64_4K, 193 VM_MODE_P36V48_4K, 194 VM_MODE_P36V48_16K, 195 VM_MODE_P36V48_64K, 196 VM_MODE_P36V47_16K, 197 NUM_VM_MODES, 198}; 199 200struct vm_shape { 201 uint32_t type; 202 uint8_t mode; 203 uint8_t subtype; 204 uint16_t padding; 205}; 206 207kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t)); 208 209#define VM_TYPE_DEFAULT 0 210 211#define VM_SHAPE(__mode) \ 212({ \ 213 struct vm_shape shape = { \ 214 .mode = (__mode), \ 215 .type = VM_TYPE_DEFAULT \ 216 }; \ 217 \ 218 shape; \ 219}) 220 221#if defined(__aarch64__) 222 223extern enum vm_guest_mode vm_mode_default; 224 225#define VM_MODE_DEFAULT vm_mode_default 226#define MIN_PAGE_SHIFT 12U 227#define ptes_per_page(page_size) ((page_size) / 8) 228 229#elif defined(__x86_64__) 230 231#define VM_MODE_DEFAULT VM_MODE_PXXV48_4K 232#define MIN_PAGE_SHIFT 12U 233#define ptes_per_page(page_size) ((page_size) / 8) 234 235#elif defined(__s390x__) 236 237#define VM_MODE_DEFAULT VM_MODE_P44V64_4K 238#define MIN_PAGE_SHIFT 12U 239#define ptes_per_page(page_size) ((page_size) / 16) 240 241#elif defined(__riscv) 242 243#if __riscv_xlen == 32 244#error "RISC-V 32-bit kvm selftests not supported" 245#endif 246 247#define VM_MODE_DEFAULT VM_MODE_P40V48_4K 248#define MIN_PAGE_SHIFT 12U 249#define ptes_per_page(page_size) ((page_size) / 8) 250 251#endif 252 253#define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) 254 255#define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) 256#define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) 257 258struct vm_guest_mode_params { 259 unsigned int pa_bits; 260 unsigned int va_bits; 261 unsigned int page_size; 262 unsigned int page_shift; 263}; 264extern const struct vm_guest_mode_params vm_guest_mode_params[]; 265 266int open_path_or_exit(const char *path, int flags); 267int open_kvm_dev_path_or_exit(void); 268 269bool get_kvm_param_bool(const char *param); 270bool get_kvm_intel_param_bool(const char *param); 271bool get_kvm_amd_param_bool(const char *param); 272 273int get_kvm_param_integer(const char *param); 274int get_kvm_intel_param_integer(const char *param); 275int get_kvm_amd_param_integer(const char *param); 276 277unsigned int kvm_check_cap(long cap); 278 279static inline bool kvm_has_cap(long cap) 280{ 281 return kvm_check_cap(cap); 282} 283 284#define __KVM_SYSCALL_ERROR(_name, _ret) \ 285 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) 286 287/* 288 * Use the "inner", double-underscore macro when reporting errors from within 289 * other macros so that the name of ioctl() and not its literal numeric value 290 * is printed on error. The "outer" macro is strongly preferred when reporting 291 * errors "directly", i.e. without an additional layer of macros, as it reduces 292 * the probability of passing in the wrong string. 293 */ 294#define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) 295#define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) 296 297#define kvm_do_ioctl(fd, cmd, arg) \ 298({ \ 299 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \ 300 ioctl(fd, cmd, arg); \ 301}) 302 303#define __kvm_ioctl(kvm_fd, cmd, arg) \ 304 kvm_do_ioctl(kvm_fd, cmd, arg) 305 306#define kvm_ioctl(kvm_fd, cmd, arg) \ 307({ \ 308 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ 309 \ 310 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ 311}) 312 313static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } 314 315#define __vm_ioctl(vm, cmd, arg) \ 316({ \ 317 static_assert_is_vm(vm); \ 318 kvm_do_ioctl((vm)->fd, cmd, arg); \ 319}) 320 321/* 322 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if 323 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, 324 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before 325 * selftests existed and (b) should never outright fail, i.e. is supposed to 326 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the 327 * VM and its vCPUs, including KVM_CHECK_EXTENSION. 328 */ 329#define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ 330do { \ 331 int __errno = errno; \ 332 \ 333 static_assert_is_vm(vm); \ 334 \ 335 if (cond) \ 336 break; \ 337 \ 338 if (errno == EIO && \ 339 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ 340 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ 341 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ 342 } \ 343 errno = __errno; \ 344 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ 345} while (0) 346 347#define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ 348 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) 349 350#define vm_ioctl(vm, cmd, arg) \ 351({ \ 352 int ret = __vm_ioctl(vm, cmd, arg); \ 353 \ 354 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ 355}) 356 357static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } 358 359#define __vcpu_ioctl(vcpu, cmd, arg) \ 360({ \ 361 static_assert_is_vcpu(vcpu); \ 362 kvm_do_ioctl((vcpu)->fd, cmd, arg); \ 363}) 364 365#define vcpu_ioctl(vcpu, cmd, arg) \ 366({ \ 367 int ret = __vcpu_ioctl(vcpu, cmd, arg); \ 368 \ 369 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ 370}) 371 372/* 373 * Looks up and returns the value corresponding to the capability 374 * (KVM_CAP_*) given by cap. 375 */ 376static inline int vm_check_cap(struct kvm_vm *vm, long cap) 377{ 378 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); 379 380 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); 381 return ret; 382} 383 384static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 385{ 386 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 387 388 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 389} 390static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 391{ 392 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 393 394 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 395} 396 397static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, 398 uint64_t size, uint64_t attributes) 399{ 400 struct kvm_memory_attributes attr = { 401 .attributes = attributes, 402 .address = gpa, 403 .size = size, 404 .flags = 0, 405 }; 406 407 /* 408 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows 409 * need significant enhancements to support multiple attributes. 410 */ 411 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, 412 "Update me to support multiple attributes!"); 413 414 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); 415} 416 417 418static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, 419 uint64_t size) 420{ 421 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 422} 423 424static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, 425 uint64_t size) 426{ 427 vm_set_memory_attributes(vm, gpa, size, 0); 428} 429 430void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, 431 bool punch_hole); 432 433static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, 434 uint64_t size) 435{ 436 vm_guest_mem_fallocate(vm, gpa, size, true); 437} 438 439static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, 440 uint64_t size) 441{ 442 vm_guest_mem_fallocate(vm, gpa, size, false); 443} 444 445void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); 446const char *vm_guest_mode_string(uint32_t i); 447 448void kvm_vm_free(struct kvm_vm *vmp); 449void kvm_vm_restart(struct kvm_vm *vmp); 450void kvm_vm_release(struct kvm_vm *vmp); 451int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva, 452 size_t len); 453void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); 454int kvm_memfd_alloc(size_t size, bool hugepages); 455 456void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 457 458static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) 459{ 460 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot }; 461 462 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args); 463} 464 465static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, 466 uint64_t first_page, uint32_t num_pages) 467{ 468 struct kvm_clear_dirty_log args = { 469 .dirty_bitmap = log, 470 .slot = slot, 471 .first_page = first_page, 472 .num_pages = num_pages 473 }; 474 475 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args); 476} 477 478static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm) 479{ 480 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL); 481} 482 483static inline int vm_get_stats_fd(struct kvm_vm *vm) 484{ 485 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); 486 487 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); 488 return fd; 489} 490 491static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header) 492{ 493 ssize_t ret; 494 495 ret = pread(stats_fd, header, sizeof(*header), 0); 496 TEST_ASSERT(ret == sizeof(*header), 497 "Failed to read '%lu' header bytes, ret = '%ld'", 498 sizeof(*header), ret); 499} 500 501struct kvm_stats_desc *read_stats_descriptors(int stats_fd, 502 struct kvm_stats_header *header); 503 504static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header) 505{ 506 /* 507 * The base size of the descriptor is defined by KVM's ABI, but the 508 * size of the name field is variable, as far as KVM's ABI is 509 * concerned. For a given instance of KVM, the name field is the same 510 * size for all stats and is provided in the overall stats header. 511 */ 512 return sizeof(struct kvm_stats_desc) + header->name_size; 513} 514 515static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats, 516 int index, 517 struct kvm_stats_header *header) 518{ 519 /* 520 * Note, size_desc includes the size of the name field, which is 521 * variable. i.e. this is NOT equivalent to &stats_desc[i]. 522 */ 523 return (void *)stats + index * get_stats_descriptor_size(header); 524} 525 526void read_stat_data(int stats_fd, struct kvm_stats_header *header, 527 struct kvm_stats_desc *desc, uint64_t *data, 528 size_t max_elements); 529 530void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, 531 size_t max_elements); 532 533static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name) 534{ 535 uint64_t data; 536 537 __vm_get_stat(vm, stat_name, &data, 1); 538 return data; 539} 540 541void vm_create_irqchip(struct kvm_vm *vm); 542 543static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 544 uint64_t flags) 545{ 546 struct kvm_create_guest_memfd guest_memfd = { 547 .size = size, 548 .flags = flags, 549 }; 550 551 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 552} 553 554static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 555 uint64_t flags) 556{ 557 int fd = __vm_create_guest_memfd(vm, size, flags); 558 559 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); 560 return fd; 561} 562 563void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 564 uint64_t gpa, uint64_t size, void *hva); 565int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 566 uint64_t gpa, uint64_t size, void *hva); 567void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 568 uint64_t gpa, uint64_t size, void *hva, 569 uint32_t guest_memfd, uint64_t guest_memfd_offset); 570int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 571 uint64_t gpa, uint64_t size, void *hva, 572 uint32_t guest_memfd, uint64_t guest_memfd_offset); 573 574void vm_userspace_mem_region_add(struct kvm_vm *vm, 575 enum vm_mem_backing_src_type src_type, 576 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 577 uint32_t flags); 578void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, 579 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 580 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); 581 582#ifndef vm_arch_has_protected_memory 583static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) 584{ 585 return false; 586} 587#endif 588 589void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); 590void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); 591void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); 592struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 593void vm_populate_vaddr_bitmap(struct kvm_vm *vm); 594vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 595vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 596vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, 597 enum kvm_mem_region_type type); 598vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz, 599 vm_vaddr_t vaddr_min, 600 enum kvm_mem_region_type type); 601vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); 602vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, 603 enum kvm_mem_region_type type); 604vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); 605 606void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 607 unsigned int npages); 608void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); 609void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); 610vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); 611void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa); 612 613 614static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa) 615{ 616 return gpa & ~vm->gpa_tag_mask; 617} 618 619void vcpu_run(struct kvm_vcpu *vcpu); 620int _vcpu_run(struct kvm_vcpu *vcpu); 621 622static inline int __vcpu_run(struct kvm_vcpu *vcpu) 623{ 624 return __vcpu_ioctl(vcpu, KVM_RUN, NULL); 625} 626 627void vcpu_run_complete_io(struct kvm_vcpu *vcpu); 628struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu); 629 630static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap, 631 uint64_t arg0) 632{ 633 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 634 635 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap); 636} 637 638static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu, 639 struct kvm_guest_debug *debug) 640{ 641 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug); 642} 643 644static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu, 645 struct kvm_mp_state *mp_state) 646{ 647 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state); 648} 649static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu, 650 struct kvm_mp_state *mp_state) 651{ 652 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state); 653} 654 655static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 656{ 657 vcpu_ioctl(vcpu, KVM_GET_REGS, regs); 658} 659 660static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 661{ 662 vcpu_ioctl(vcpu, KVM_SET_REGS, regs); 663} 664static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 665{ 666 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs); 667 668} 669static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 670{ 671 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 672} 673static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 674{ 675 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 676} 677static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 678{ 679 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu); 680} 681static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 682{ 683 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu); 684} 685 686static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 687{ 688 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 689 690 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 691} 692static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 693{ 694 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 695 696 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 697} 698static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 699{ 700 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 701 702 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 703} 704static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 705{ 706 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 707 708 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 709} 710 711#ifdef __KVM_HAVE_VCPU_EVENTS 712static inline void vcpu_events_get(struct kvm_vcpu *vcpu, 713 struct kvm_vcpu_events *events) 714{ 715 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events); 716} 717static inline void vcpu_events_set(struct kvm_vcpu *vcpu, 718 struct kvm_vcpu_events *events) 719{ 720 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events); 721} 722#endif 723#ifdef __x86_64__ 724static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu, 725 struct kvm_nested_state *state) 726{ 727 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state); 728} 729static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu, 730 struct kvm_nested_state *state) 731{ 732 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 733} 734 735static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu, 736 struct kvm_nested_state *state) 737{ 738 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 739} 740#endif 741static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) 742{ 743 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); 744 745 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); 746 return fd; 747} 748 749int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr); 750 751static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr) 752{ 753 int ret = __kvm_has_device_attr(dev_fd, group, attr); 754 755 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno); 756} 757 758int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val); 759 760static inline void kvm_device_attr_get(int dev_fd, uint32_t group, 761 uint64_t attr, void *val) 762{ 763 int ret = __kvm_device_attr_get(dev_fd, group, attr, val); 764 765 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret)); 766} 767 768int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val); 769 770static inline void kvm_device_attr_set(int dev_fd, uint32_t group, 771 uint64_t attr, void *val) 772{ 773 int ret = __kvm_device_attr_set(dev_fd, group, attr, val); 774 775 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 776} 777 778static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 779 uint64_t attr) 780{ 781 return __kvm_has_device_attr(vcpu->fd, group, attr); 782} 783 784static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 785 uint64_t attr) 786{ 787 kvm_has_device_attr(vcpu->fd, group, attr); 788} 789 790static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 791 uint64_t attr, void *val) 792{ 793 return __kvm_device_attr_get(vcpu->fd, group, attr, val); 794} 795 796static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 797 uint64_t attr, void *val) 798{ 799 kvm_device_attr_get(vcpu->fd, group, attr, val); 800} 801 802static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 803 uint64_t attr, void *val) 804{ 805 return __kvm_device_attr_set(vcpu->fd, group, attr, val); 806} 807 808static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 809 uint64_t attr, void *val) 810{ 811 kvm_device_attr_set(vcpu->fd, group, attr, val); 812} 813 814int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type); 815int __kvm_create_device(struct kvm_vm *vm, uint64_t type); 816 817static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type) 818{ 819 int fd = __kvm_create_device(vm, type); 820 821 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd)); 822 return fd; 823} 824 825void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); 826 827/* 828 * VM VCPU Args Set 829 * 830 * Input Args: 831 * vm - Virtual Machine 832 * num - number of arguments 833 * ... - arguments, each of type uint64_t 834 * 835 * Output Args: None 836 * 837 * Return: None 838 * 839 * Sets the first @num input parameters for the function at @vcpu's entry point, 840 * per the C calling convention of the architecture, to the values given as 841 * variable args. Each of the variable args is expected to be of type uint64_t. 842 * The maximum @num can be is specific to the architecture. 843 */ 844void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); 845 846void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 847int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 848 849#define KVM_MAX_IRQ_ROUTES 4096 850 851struct kvm_irq_routing *kvm_gsi_routing_create(void); 852void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, 853 uint32_t gsi, uint32_t pin); 854int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 855void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 856 857const char *exit_reason_str(unsigned int exit_reason); 858 859vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, 860 uint32_t memslot); 861vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 862 vm_paddr_t paddr_min, uint32_t memslot, 863 bool protected); 864vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); 865 866static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 867 vm_paddr_t paddr_min, uint32_t memslot) 868{ 869 /* 870 * By default, allocate memory as protected for VMs that support 871 * protected memory, as the majority of memory for such VMs is 872 * protected, i.e. using shared memory is effectively opt-in. 873 */ 874 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot, 875 vm_arch_has_protected_memory(vm)); 876} 877 878/* 879 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also 880 * loads the test binary into guest memory and creates an IRQ chip (x86 only). 881 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to 882 * calculate the amount of memory needed for per-vCPU data, e.g. stacks. 883 */ 884struct kvm_vm *____vm_create(struct vm_shape shape); 885struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, 886 uint64_t nr_extra_pages); 887 888static inline struct kvm_vm *vm_create_barebones(void) 889{ 890 return ____vm_create(VM_SHAPE_DEFAULT); 891} 892 893#ifdef __x86_64__ 894static inline struct kvm_vm *vm_create_barebones_protected_vm(void) 895{ 896 const struct vm_shape shape = { 897 .mode = VM_MODE_DEFAULT, 898 .type = KVM_X86_SW_PROTECTED_VM, 899 }; 900 901 return ____vm_create(shape); 902} 903#endif 904 905static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) 906{ 907 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); 908} 909 910struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, 911 uint64_t extra_mem_pages, 912 void *guest_code, struct kvm_vcpu *vcpus[]); 913 914static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, 915 void *guest_code, 916 struct kvm_vcpu *vcpus[]) 917{ 918 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, 919 guest_code, vcpus); 920} 921 922 923struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, 924 struct kvm_vcpu **vcpu, 925 uint64_t extra_mem_pages, 926 void *guest_code); 927 928/* 929 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages 930 * additional pages of guest memory. Returns the VM and vCPU (via out param). 931 */ 932static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 933 uint64_t extra_mem_pages, 934 void *guest_code) 935{ 936 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, 937 extra_mem_pages, guest_code); 938} 939 940static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 941 void *guest_code) 942{ 943 return __vm_create_with_one_vcpu(vcpu, 0, guest_code); 944} 945 946static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, 947 struct kvm_vcpu **vcpu, 948 void *guest_code) 949{ 950 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); 951} 952 953struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); 954 955void kvm_pin_this_task_to_pcpu(uint32_t pcpu); 956void kvm_print_vcpu_pinning_help(void); 957void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[], 958 int nr_vcpus); 959 960unsigned long vm_compute_max_gfn(struct kvm_vm *vm); 961unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); 962unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); 963unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); 964static inline unsigned int 965vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) 966{ 967 unsigned int n; 968 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); 969#ifdef __s390x__ 970 /* s390 requires 1M aligned guest sizes */ 971 n = (n + 255) & ~255; 972#endif 973 return n; 974} 975 976#define sync_global_to_guest(vm, g) ({ \ 977 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 978 memcpy(_p, &(g), sizeof(g)); \ 979}) 980 981#define sync_global_from_guest(vm, g) ({ \ 982 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 983 memcpy(&(g), _p, sizeof(g)); \ 984}) 985 986/* 987 * Write a global value, but only in the VM's (guest's) domain. Primarily used 988 * for "globals" that hold per-VM values (VMs always duplicate code and global 989 * data into their own region of physical memory), but can be used anytime it's 990 * undesirable to change the host's copy of the global. 991 */ 992#define write_guest_global(vm, g, val) ({ \ 993 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 994 typeof(g) _val = val; \ 995 \ 996 memcpy(_p, &(_val), sizeof(g)); \ 997}) 998 999void assert_on_unhandled_exception(struct kvm_vcpu *vcpu); 1000 1001void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, 1002 uint8_t indent); 1003 1004static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu, 1005 uint8_t indent) 1006{ 1007 vcpu_arch_dump(stream, vcpu, indent); 1008} 1009 1010/* 1011 * Adds a vCPU with reasonable defaults (e.g. a stack) 1012 * 1013 * Input Args: 1014 * vm - Virtual Machine 1015 * vcpu_id - The id of the VCPU to add to the VM. 1016 */ 1017struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 1018void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code); 1019 1020static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 1021 void *guest_code) 1022{ 1023 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id); 1024 1025 vcpu_arch_set_entry_point(vcpu, guest_code); 1026 1027 return vcpu; 1028} 1029 1030/* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */ 1031struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id); 1032 1033static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm, 1034 uint32_t vcpu_id) 1035{ 1036 return vm_arch_vcpu_recreate(vm, vcpu_id); 1037} 1038 1039void vcpu_arch_free(struct kvm_vcpu *vcpu); 1040 1041void virt_arch_pgd_alloc(struct kvm_vm *vm); 1042 1043static inline void virt_pgd_alloc(struct kvm_vm *vm) 1044{ 1045 virt_arch_pgd_alloc(vm); 1046} 1047 1048/* 1049 * VM Virtual Page Map 1050 * 1051 * Input Args: 1052 * vm - Virtual Machine 1053 * vaddr - VM Virtual Address 1054 * paddr - VM Physical Address 1055 * memslot - Memory region slot for new virtual translation tables 1056 * 1057 * Output Args: None 1058 * 1059 * Return: None 1060 * 1061 * Within @vm, creates a virtual translation for the page starting 1062 * at @vaddr to the page starting at @paddr. 1063 */ 1064void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); 1065 1066static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 1067{ 1068 virt_arch_pg_map(vm, vaddr, paddr); 1069} 1070 1071 1072/* 1073 * Address Guest Virtual to Guest Physical 1074 * 1075 * Input Args: 1076 * vm - Virtual Machine 1077 * gva - VM virtual address 1078 * 1079 * Output Args: None 1080 * 1081 * Return: 1082 * Equivalent VM physical address 1083 * 1084 * Returns the VM physical address of the translated VM virtual 1085 * address given by @gva. 1086 */ 1087vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); 1088 1089static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 1090{ 1091 return addr_arch_gva2gpa(vm, gva); 1092} 1093 1094/* 1095 * Virtual Translation Tables Dump 1096 * 1097 * Input Args: 1098 * stream - Output FILE stream 1099 * vm - Virtual Machine 1100 * indent - Left margin indent amount 1101 * 1102 * Output Args: None 1103 * 1104 * Return: None 1105 * 1106 * Dumps to the FILE stream given by @stream, the contents of all the 1107 * virtual translation tables for the VM given by @vm. 1108 */ 1109void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 1110 1111static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 1112{ 1113 virt_arch_dump(stream, vm, indent); 1114} 1115 1116 1117static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) 1118{ 1119 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0); 1120} 1121 1122/* 1123 * Arch hook that is invoked via a constructor, i.e. before exeucting main(), 1124 * to allow for arch-specific setup that is common to all tests, e.g. computing 1125 * the default guest "mode". 1126 */ 1127void kvm_selftest_arch_init(void); 1128 1129void kvm_arch_vm_post_create(struct kvm_vm *vm); 1130 1131bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1132 1133uint32_t guest_get_vcpuid(void); 1134 1135#endif /* SELFTEST_KVM_UTIL_BASE_H */ 1136