1// SPDX-License-Identifier: GPL-2.0 2/* 3 * page_fault_test.c - Test stage 2 faults. 4 * 5 * This test tries different combinations of guest accesses (e.g., write, 6 * S1PTW), backing source type (e.g., anon) and types of faults (e.g., read on 7 * hugetlbfs with a hole). It checks that the expected handling method is 8 * called (e.g., uffd faults with the right address and write/read flag). 9 */ 10#define _GNU_SOURCE 11#include <linux/bitmap.h> 12#include <fcntl.h> 13#include <test_util.h> 14#include <kvm_util.h> 15#include <processor.h> 16#include <asm/sysreg.h> 17#include <linux/bitfield.h> 18#include "guest_modes.h" 19#include "userfaultfd_util.h" 20 21/* Guest virtual addresses that point to the test page and its PTE. */ 22#define TEST_GVA 0xc0000000 23#define TEST_EXEC_GVA (TEST_GVA + 0x8) 24#define TEST_PTE_GVA 0xb0000000 25#define TEST_DATA 0x0123456789ABCDEF 26 27static uint64_t *guest_test_memory = (uint64_t *)TEST_GVA; 28 29#define CMD_NONE (0) 30#define CMD_SKIP_TEST (1ULL << 1) 31#define CMD_HOLE_PT (1ULL << 2) 32#define CMD_HOLE_DATA (1ULL << 3) 33#define CMD_CHECK_WRITE_IN_DIRTY_LOG (1ULL << 4) 34#define CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG (1ULL << 5) 35#define CMD_CHECK_NO_WRITE_IN_DIRTY_LOG (1ULL << 6) 36#define CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG (1ULL << 7) 37#define CMD_SET_PTE_AF (1ULL << 8) 38 39#define PREPARE_FN_NR 10 40#define CHECK_FN_NR 10 41 42static struct event_cnt { 43 int mmio_exits; 44 int fail_vcpu_runs; 45 int uffd_faults; 46 /* uffd_faults is incremented from multiple threads. */ 47 pthread_mutex_t uffd_faults_mutex; 48} events; 49 50struct test_desc { 51 const char *name; 52 uint64_t mem_mark_cmd; 53 /* Skip the test if any prepare function returns false */ 54 bool (*guest_prepare[PREPARE_FN_NR])(void); 55 void (*guest_test)(void); 56 void (*guest_test_check[CHECK_FN_NR])(void); 57 uffd_handler_t uffd_pt_handler; 58 uffd_handler_t uffd_data_handler; 59 void (*dabt_handler)(struct ex_regs *regs); 60 void (*iabt_handler)(struct ex_regs *regs); 61 void (*mmio_handler)(struct kvm_vm *vm, struct kvm_run *run); 62 void (*fail_vcpu_run_handler)(int ret); 63 uint32_t pt_memslot_flags; 64 uint32_t data_memslot_flags; 65 bool skip; 66 struct event_cnt expected_events; 67}; 68 69struct test_params { 70 enum vm_mem_backing_src_type src_type; 71 struct test_desc *test_desc; 72}; 73 74static inline void flush_tlb_page(uint64_t vaddr) 75{ 76 uint64_t page = vaddr >> 12; 77 78 dsb(ishst); 79 asm volatile("tlbi vaae1is, %0" :: "r" (page)); 80 dsb(ish); 81 isb(); 82} 83 84static void guest_write64(void) 85{ 86 uint64_t val; 87 88 WRITE_ONCE(*guest_test_memory, TEST_DATA); 89 val = READ_ONCE(*guest_test_memory); 90 GUEST_ASSERT_EQ(val, TEST_DATA); 91} 92 93/* Check the system for atomic instructions. */ 94static bool guest_check_lse(void) 95{ 96 uint64_t isar0 = read_sysreg(id_aa64isar0_el1); 97 uint64_t atomic; 98 99 atomic = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_ATOMIC), isar0); 100 return atomic >= 2; 101} 102 103static bool guest_check_dc_zva(void) 104{ 105 uint64_t dczid = read_sysreg(dczid_el0); 106 uint64_t dzp = FIELD_GET(ARM64_FEATURE_MASK(DCZID_EL0_DZP), dczid); 107 108 return dzp == 0; 109} 110 111/* Compare and swap instruction. */ 112static void guest_cas(void) 113{ 114 uint64_t val; 115 116 GUEST_ASSERT(guest_check_lse()); 117 asm volatile(".arch_extension lse\n" 118 "casal %0, %1, [%2]\n" 119 :: "r" (0ul), "r" (TEST_DATA), "r" (guest_test_memory)); 120 val = READ_ONCE(*guest_test_memory); 121 GUEST_ASSERT_EQ(val, TEST_DATA); 122} 123 124static void guest_read64(void) 125{ 126 uint64_t val; 127 128 val = READ_ONCE(*guest_test_memory); 129 GUEST_ASSERT_EQ(val, 0); 130} 131 132/* Address translation instruction */ 133static void guest_at(void) 134{ 135 uint64_t par; 136 137 asm volatile("at s1e1r, %0" :: "r" (guest_test_memory)); 138 isb(); 139 par = read_sysreg(par_el1); 140 141 /* Bit 1 indicates whether the AT was successful */ 142 GUEST_ASSERT_EQ(par & 1, 0); 143} 144 145/* 146 * The size of the block written by "dc zva" is guaranteed to be between (2 << 147 * 0) and (2 << 9), which is safe in our case as we need the write to happen 148 * for at least a word, and not more than a page. 149 */ 150static void guest_dc_zva(void) 151{ 152 uint16_t val; 153 154 asm volatile("dc zva, %0" :: "r" (guest_test_memory)); 155 dsb(ish); 156 val = READ_ONCE(*guest_test_memory); 157 GUEST_ASSERT_EQ(val, 0); 158} 159 160/* 161 * Pre-indexing loads and stores don't have a valid syndrome (ESR_EL2.ISV==0). 162 * And that's special because KVM must take special care with those: they 163 * should still count as accesses for dirty logging or user-faulting, but 164 * should be handled differently on mmio. 165 */ 166static void guest_ld_preidx(void) 167{ 168 uint64_t val; 169 uint64_t addr = TEST_GVA - 8; 170 171 /* 172 * This ends up accessing "TEST_GVA + 8 - 8", where "TEST_GVA - 8" is 173 * in a gap between memslots not backing by anything. 174 */ 175 asm volatile("ldr %0, [%1, #8]!" 176 : "=r" (val), "+r" (addr)); 177 GUEST_ASSERT_EQ(val, 0); 178 GUEST_ASSERT_EQ(addr, TEST_GVA); 179} 180 181static void guest_st_preidx(void) 182{ 183 uint64_t val = TEST_DATA; 184 uint64_t addr = TEST_GVA - 8; 185 186 asm volatile("str %0, [%1, #8]!" 187 : "+r" (val), "+r" (addr)); 188 189 GUEST_ASSERT_EQ(addr, TEST_GVA); 190 val = READ_ONCE(*guest_test_memory); 191} 192 193static bool guest_set_ha(void) 194{ 195 uint64_t mmfr1 = read_sysreg(id_aa64mmfr1_el1); 196 uint64_t hadbs, tcr; 197 198 /* Skip if HA is not supported. */ 199 hadbs = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS), mmfr1); 200 if (hadbs == 0) 201 return false; 202 203 tcr = read_sysreg(tcr_el1) | TCR_EL1_HA; 204 write_sysreg(tcr, tcr_el1); 205 isb(); 206 207 return true; 208} 209 210static bool guest_clear_pte_af(void) 211{ 212 *((uint64_t *)TEST_PTE_GVA) &= ~PTE_AF; 213 flush_tlb_page(TEST_GVA); 214 215 return true; 216} 217 218static void guest_check_pte_af(void) 219{ 220 dsb(ish); 221 GUEST_ASSERT_EQ(*((uint64_t *)TEST_PTE_GVA) & PTE_AF, PTE_AF); 222} 223 224static void guest_check_write_in_dirty_log(void) 225{ 226 GUEST_SYNC(CMD_CHECK_WRITE_IN_DIRTY_LOG); 227} 228 229static void guest_check_no_write_in_dirty_log(void) 230{ 231 GUEST_SYNC(CMD_CHECK_NO_WRITE_IN_DIRTY_LOG); 232} 233 234static void guest_check_s1ptw_wr_in_dirty_log(void) 235{ 236 GUEST_SYNC(CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG); 237} 238 239static void guest_check_no_s1ptw_wr_in_dirty_log(void) 240{ 241 GUEST_SYNC(CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG); 242} 243 244static void guest_exec(void) 245{ 246 int (*code)(void) = (int (*)(void))TEST_EXEC_GVA; 247 int ret; 248 249 ret = code(); 250 GUEST_ASSERT_EQ(ret, 0x77); 251} 252 253static bool guest_prepare(struct test_desc *test) 254{ 255 bool (*prepare_fn)(void); 256 int i; 257 258 for (i = 0; i < PREPARE_FN_NR; i++) { 259 prepare_fn = test->guest_prepare[i]; 260 if (prepare_fn && !prepare_fn()) 261 return false; 262 } 263 264 return true; 265} 266 267static void guest_test_check(struct test_desc *test) 268{ 269 void (*check_fn)(void); 270 int i; 271 272 for (i = 0; i < CHECK_FN_NR; i++) { 273 check_fn = test->guest_test_check[i]; 274 if (check_fn) 275 check_fn(); 276 } 277} 278 279static void guest_code(struct test_desc *test) 280{ 281 if (!guest_prepare(test)) 282 GUEST_SYNC(CMD_SKIP_TEST); 283 284 GUEST_SYNC(test->mem_mark_cmd); 285 286 if (test->guest_test) 287 test->guest_test(); 288 289 guest_test_check(test); 290 GUEST_DONE(); 291} 292 293static void no_dabt_handler(struct ex_regs *regs) 294{ 295 GUEST_FAIL("Unexpected dabt, far_el1 = 0x%lx", read_sysreg(far_el1)); 296} 297 298static void no_iabt_handler(struct ex_regs *regs) 299{ 300 GUEST_FAIL("Unexpected iabt, pc = 0x%lx", regs->pc); 301} 302 303static struct uffd_args { 304 char *copy; 305 void *hva; 306 uint64_t paging_size; 307} pt_args, data_args; 308 309/* Returns true to continue the test, and false if it should be skipped. */ 310static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg, 311 struct uffd_args *args) 312{ 313 uint64_t addr = msg->arg.pagefault.address; 314 uint64_t flags = msg->arg.pagefault.flags; 315 struct uffdio_copy copy; 316 int ret; 317 318 TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING, 319 "The only expected UFFD mode is MISSING"); 320 TEST_ASSERT_EQ(addr, (uint64_t)args->hva); 321 322 pr_debug("uffd fault: addr=%p write=%d\n", 323 (void *)addr, !!(flags & UFFD_PAGEFAULT_FLAG_WRITE)); 324 325 copy.src = (uint64_t)args->copy; 326 copy.dst = addr; 327 copy.len = args->paging_size; 328 copy.mode = 0; 329 330 ret = ioctl(uffd, UFFDIO_COPY, ©); 331 if (ret == -1) { 332 pr_info("Failed UFFDIO_COPY in 0x%lx with errno: %d\n", 333 addr, errno); 334 return ret; 335 } 336 337 pthread_mutex_lock(&events.uffd_faults_mutex); 338 events.uffd_faults += 1; 339 pthread_mutex_unlock(&events.uffd_faults_mutex); 340 return 0; 341} 342 343static int uffd_pt_handler(int mode, int uffd, struct uffd_msg *msg) 344{ 345 return uffd_generic_handler(mode, uffd, msg, &pt_args); 346} 347 348static int uffd_data_handler(int mode, int uffd, struct uffd_msg *msg) 349{ 350 return uffd_generic_handler(mode, uffd, msg, &data_args); 351} 352 353static void setup_uffd_args(struct userspace_mem_region *region, 354 struct uffd_args *args) 355{ 356 args->hva = (void *)region->region.userspace_addr; 357 args->paging_size = region->region.memory_size; 358 359 args->copy = malloc(args->paging_size); 360 TEST_ASSERT(args->copy, "Failed to allocate data copy."); 361 memcpy(args->copy, args->hva, args->paging_size); 362} 363 364static void setup_uffd(struct kvm_vm *vm, struct test_params *p, 365 struct uffd_desc **pt_uffd, struct uffd_desc **data_uffd) 366{ 367 struct test_desc *test = p->test_desc; 368 int uffd_mode = UFFDIO_REGISTER_MODE_MISSING; 369 370 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_PT), &pt_args); 371 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_TEST_DATA), &data_args); 372 373 *pt_uffd = NULL; 374 if (test->uffd_pt_handler) 375 *pt_uffd = uffd_setup_demand_paging(uffd_mode, 0, 376 pt_args.hva, 377 pt_args.paging_size, 378 test->uffd_pt_handler); 379 380 *data_uffd = NULL; 381 if (test->uffd_data_handler) 382 *data_uffd = uffd_setup_demand_paging(uffd_mode, 0, 383 data_args.hva, 384 data_args.paging_size, 385 test->uffd_data_handler); 386} 387 388static void free_uffd(struct test_desc *test, struct uffd_desc *pt_uffd, 389 struct uffd_desc *data_uffd) 390{ 391 if (test->uffd_pt_handler) 392 uffd_stop_demand_paging(pt_uffd); 393 if (test->uffd_data_handler) 394 uffd_stop_demand_paging(data_uffd); 395 396 free(pt_args.copy); 397 free(data_args.copy); 398} 399 400static int uffd_no_handler(int mode, int uffd, struct uffd_msg *msg) 401{ 402 TEST_FAIL("There was no UFFD fault expected."); 403 return -1; 404} 405 406/* Returns false if the test should be skipped. */ 407static bool punch_hole_in_backing_store(struct kvm_vm *vm, 408 struct userspace_mem_region *region) 409{ 410 void *hva = (void *)region->region.userspace_addr; 411 uint64_t paging_size = region->region.memory_size; 412 int ret, fd = region->fd; 413 414 if (fd != -1) { 415 ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 416 0, paging_size); 417 TEST_ASSERT(ret == 0, "fallocate failed"); 418 } else { 419 ret = madvise(hva, paging_size, MADV_DONTNEED); 420 TEST_ASSERT(ret == 0, "madvise failed"); 421 } 422 423 return true; 424} 425 426static void mmio_on_test_gpa_handler(struct kvm_vm *vm, struct kvm_run *run) 427{ 428 struct userspace_mem_region *region; 429 void *hva; 430 431 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); 432 hva = (void *)region->region.userspace_addr; 433 434 TEST_ASSERT_EQ(run->mmio.phys_addr, region->region.guest_phys_addr); 435 436 memcpy(hva, run->mmio.data, run->mmio.len); 437 events.mmio_exits += 1; 438} 439 440static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run) 441{ 442 uint64_t data; 443 444 memcpy(&data, run->mmio.data, sizeof(data)); 445 pr_debug("addr=%lld len=%d w=%d data=%lx\n", 446 run->mmio.phys_addr, run->mmio.len, 447 run->mmio.is_write, data); 448 TEST_FAIL("There was no MMIO exit expected."); 449} 450 451static bool check_write_in_dirty_log(struct kvm_vm *vm, 452 struct userspace_mem_region *region, 453 uint64_t host_pg_nr) 454{ 455 unsigned long *bmap; 456 bool first_page_dirty; 457 uint64_t size = region->region.memory_size; 458 459 /* getpage_size() is not always equal to vm->page_size */ 460 bmap = bitmap_zalloc(size / getpagesize()); 461 kvm_vm_get_dirty_log(vm, region->region.slot, bmap); 462 first_page_dirty = test_bit(host_pg_nr, bmap); 463 free(bmap); 464 return first_page_dirty; 465} 466 467/* Returns true to continue the test, and false if it should be skipped. */ 468static bool handle_cmd(struct kvm_vm *vm, int cmd) 469{ 470 struct userspace_mem_region *data_region, *pt_region; 471 bool continue_test = true; 472 uint64_t pte_gpa, pte_pg; 473 474 data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); 475 pt_region = vm_get_mem_region(vm, MEM_REGION_PT); 476 pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA)); 477 pte_pg = (pte_gpa - pt_region->region.guest_phys_addr) / getpagesize(); 478 479 if (cmd == CMD_SKIP_TEST) 480 continue_test = false; 481 482 if (cmd & CMD_HOLE_PT) 483 continue_test = punch_hole_in_backing_store(vm, pt_region); 484 if (cmd & CMD_HOLE_DATA) 485 continue_test = punch_hole_in_backing_store(vm, data_region); 486 if (cmd & CMD_CHECK_WRITE_IN_DIRTY_LOG) 487 TEST_ASSERT(check_write_in_dirty_log(vm, data_region, 0), 488 "Missing write in dirty log"); 489 if (cmd & CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG) 490 TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, pte_pg), 491 "Missing s1ptw write in dirty log"); 492 if (cmd & CMD_CHECK_NO_WRITE_IN_DIRTY_LOG) 493 TEST_ASSERT(!check_write_in_dirty_log(vm, data_region, 0), 494 "Unexpected write in dirty log"); 495 if (cmd & CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG) 496 TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, pte_pg), 497 "Unexpected s1ptw write in dirty log"); 498 499 return continue_test; 500} 501 502void fail_vcpu_run_no_handler(int ret) 503{ 504 TEST_FAIL("Unexpected vcpu run failure"); 505} 506 507void fail_vcpu_run_mmio_no_syndrome_handler(int ret) 508{ 509 TEST_ASSERT(errno == ENOSYS, 510 "The mmio handler should have returned not implemented."); 511 events.fail_vcpu_runs += 1; 512} 513 514typedef uint32_t aarch64_insn_t; 515extern aarch64_insn_t __exec_test[2]; 516 517noinline void __return_0x77(void) 518{ 519 asm volatile("__exec_test: mov x0, #0x77\n" 520 "ret\n"); 521} 522 523/* 524 * Note that this function runs on the host before the test VM starts: there's 525 * no need to sync the D$ and I$ caches. 526 */ 527static void load_exec_code_for_test(struct kvm_vm *vm) 528{ 529 uint64_t *code; 530 struct userspace_mem_region *region; 531 void *hva; 532 533 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); 534 hva = (void *)region->region.userspace_addr; 535 536 assert(TEST_EXEC_GVA > TEST_GVA); 537 code = hva + TEST_EXEC_GVA - TEST_GVA; 538 memcpy(code, __exec_test, sizeof(__exec_test)); 539} 540 541static void setup_abort_handlers(struct kvm_vm *vm, struct kvm_vcpu *vcpu, 542 struct test_desc *test) 543{ 544 vm_init_descriptor_tables(vm); 545 vcpu_init_descriptor_tables(vcpu); 546 547 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, 548 ESR_EC_DABT, no_dabt_handler); 549 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, 550 ESR_EC_IABT, no_iabt_handler); 551} 552 553static void setup_gva_maps(struct kvm_vm *vm) 554{ 555 struct userspace_mem_region *region; 556 uint64_t pte_gpa; 557 558 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); 559 /* Map TEST_GVA first. This will install a new PTE. */ 560 virt_pg_map(vm, TEST_GVA, region->region.guest_phys_addr); 561 /* Then map TEST_PTE_GVA to the above PTE. */ 562 pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA)); 563 virt_pg_map(vm, TEST_PTE_GVA, pte_gpa); 564} 565 566enum pf_test_memslots { 567 CODE_AND_DATA_MEMSLOT, 568 PAGE_TABLE_MEMSLOT, 569 TEST_DATA_MEMSLOT, 570}; 571 572/* 573 * Create a memslot for code and data at pfn=0, and test-data and PT ones 574 * at max_gfn. 575 */ 576static void setup_memslots(struct kvm_vm *vm, struct test_params *p) 577{ 578 uint64_t backing_src_pagesz = get_backing_src_pagesz(p->src_type); 579 uint64_t guest_page_size = vm->page_size; 580 uint64_t max_gfn = vm_compute_max_gfn(vm); 581 /* Enough for 2M of code when using 4K guest pages. */ 582 uint64_t code_npages = 512; 583 uint64_t pt_size, data_size, data_gpa; 584 585 /* 586 * This test requires 1 pgd, 2 pud, 4 pmd, and 6 pte pages when using 587 * VM_MODE_P48V48_4K. Note that the .text takes ~1.6MBs. That's 13 588 * pages. VM_MODE_P48V48_4K is the mode with most PT pages; let's use 589 * twice that just in case. 590 */ 591 pt_size = 26 * guest_page_size; 592 593 /* memslot sizes and gpa's must be aligned to the backing page size */ 594 pt_size = align_up(pt_size, backing_src_pagesz); 595 data_size = align_up(guest_page_size, backing_src_pagesz); 596 data_gpa = (max_gfn * guest_page_size) - data_size; 597 data_gpa = align_down(data_gpa, backing_src_pagesz); 598 599 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 600 CODE_AND_DATA_MEMSLOT, code_npages, 0); 601 vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT; 602 vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT; 603 604 vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size, 605 PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, 606 p->test_desc->pt_memslot_flags); 607 vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT; 608 609 vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT, 610 data_size / guest_page_size, 611 p->test_desc->data_memslot_flags); 612 vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; 613} 614 615static void setup_ucall(struct kvm_vm *vm) 616{ 617 struct userspace_mem_region *region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); 618 619 ucall_init(vm, region->region.guest_phys_addr + region->region.memory_size); 620} 621 622static void setup_default_handlers(struct test_desc *test) 623{ 624 if (!test->mmio_handler) 625 test->mmio_handler = mmio_no_handler; 626 627 if (!test->fail_vcpu_run_handler) 628 test->fail_vcpu_run_handler = fail_vcpu_run_no_handler; 629} 630 631static void check_event_counts(struct test_desc *test) 632{ 633 TEST_ASSERT_EQ(test->expected_events.uffd_faults, events.uffd_faults); 634 TEST_ASSERT_EQ(test->expected_events.mmio_exits, events.mmio_exits); 635 TEST_ASSERT_EQ(test->expected_events.fail_vcpu_runs, events.fail_vcpu_runs); 636} 637 638static void print_test_banner(enum vm_guest_mode mode, struct test_params *p) 639{ 640 struct test_desc *test = p->test_desc; 641 642 pr_debug("Test: %s\n", test->name); 643 pr_debug("Testing guest mode: %s\n", vm_guest_mode_string(mode)); 644 pr_debug("Testing memory backing src type: %s\n", 645 vm_mem_backing_src_alias(p->src_type)->name); 646} 647 648static void reset_event_counts(void) 649{ 650 memset(&events, 0, sizeof(events)); 651} 652 653/* 654 * This function either succeeds, skips the test (after setting test->skip), or 655 * fails with a TEST_FAIL that aborts all tests. 656 */ 657static void vcpu_run_loop(struct kvm_vm *vm, struct kvm_vcpu *vcpu, 658 struct test_desc *test) 659{ 660 struct kvm_run *run; 661 struct ucall uc; 662 int ret; 663 664 run = vcpu->run; 665 666 for (;;) { 667 ret = _vcpu_run(vcpu); 668 if (ret) { 669 test->fail_vcpu_run_handler(ret); 670 goto done; 671 } 672 673 switch (get_ucall(vcpu, &uc)) { 674 case UCALL_SYNC: 675 if (!handle_cmd(vm, uc.args[1])) { 676 test->skip = true; 677 goto done; 678 } 679 break; 680 case UCALL_ABORT: 681 REPORT_GUEST_ASSERT(uc); 682 break; 683 case UCALL_DONE: 684 goto done; 685 case UCALL_NONE: 686 if (run->exit_reason == KVM_EXIT_MMIO) 687 test->mmio_handler(vm, run); 688 break; 689 default: 690 TEST_FAIL("Unknown ucall %lu", uc.cmd); 691 } 692 } 693 694done: 695 pr_debug(test->skip ? "Skipped.\n" : "Done.\n"); 696} 697 698static void run_test(enum vm_guest_mode mode, void *arg) 699{ 700 struct test_params *p = (struct test_params *)arg; 701 struct test_desc *test = p->test_desc; 702 struct kvm_vm *vm; 703 struct kvm_vcpu *vcpu; 704 struct uffd_desc *pt_uffd, *data_uffd; 705 706 print_test_banner(mode, p); 707 708 vm = ____vm_create(VM_SHAPE(mode)); 709 setup_memslots(vm, p); 710 kvm_vm_elf_load(vm, program_invocation_name); 711 setup_ucall(vm); 712 vcpu = vm_vcpu_add(vm, 0, guest_code); 713 714 setup_gva_maps(vm); 715 716 reset_event_counts(); 717 718 /* 719 * Set some code in the data memslot for the guest to execute (only 720 * applicable to the EXEC tests). This has to be done before 721 * setup_uffd() as that function copies the memslot data for the uffd 722 * handler. 723 */ 724 load_exec_code_for_test(vm); 725 setup_uffd(vm, p, &pt_uffd, &data_uffd); 726 setup_abort_handlers(vm, vcpu, test); 727 setup_default_handlers(test); 728 vcpu_args_set(vcpu, 1, test); 729 730 vcpu_run_loop(vm, vcpu, test); 731 732 kvm_vm_free(vm); 733 free_uffd(test, pt_uffd, data_uffd); 734 735 /* 736 * Make sure we check the events after the uffd threads have exited, 737 * which means they updated their respective event counters. 738 */ 739 if (!test->skip) 740 check_event_counts(test); 741} 742 743static void help(char *name) 744{ 745 puts(""); 746 printf("usage: %s [-h] [-s mem-type]\n", name); 747 puts(""); 748 guest_modes_help(); 749 backing_src_help("-s"); 750 puts(""); 751} 752 753#define SNAME(s) #s 754#define SCAT2(a, b) SNAME(a ## _ ## b) 755#define SCAT3(a, b, c) SCAT2(a, SCAT2(b, c)) 756#define SCAT4(a, b, c, d) SCAT2(a, SCAT3(b, c, d)) 757 758#define _CHECK(_test) _CHECK_##_test 759#define _PREPARE(_test) _PREPARE_##_test 760#define _PREPARE_guest_read64 NULL 761#define _PREPARE_guest_ld_preidx NULL 762#define _PREPARE_guest_write64 NULL 763#define _PREPARE_guest_st_preidx NULL 764#define _PREPARE_guest_exec NULL 765#define _PREPARE_guest_at NULL 766#define _PREPARE_guest_dc_zva guest_check_dc_zva 767#define _PREPARE_guest_cas guest_check_lse 768 769/* With or without access flag checks */ 770#define _PREPARE_with_af guest_set_ha, guest_clear_pte_af 771#define _PREPARE_no_af NULL 772#define _CHECK_with_af guest_check_pte_af 773#define _CHECK_no_af NULL 774 775/* Performs an access and checks that no faults were triggered. */ 776#define TEST_ACCESS(_access, _with_af, _mark_cmd) \ 777{ \ 778 .name = SCAT3(_access, _with_af, #_mark_cmd), \ 779 .guest_prepare = { _PREPARE(_with_af), \ 780 _PREPARE(_access) }, \ 781 .mem_mark_cmd = _mark_cmd, \ 782 .guest_test = _access, \ 783 .guest_test_check = { _CHECK(_with_af) }, \ 784 .expected_events = { 0 }, \ 785} 786 787#define TEST_UFFD(_access, _with_af, _mark_cmd, \ 788 _uffd_data_handler, _uffd_pt_handler, _uffd_faults) \ 789{ \ 790 .name = SCAT4(uffd, _access, _with_af, #_mark_cmd), \ 791 .guest_prepare = { _PREPARE(_with_af), \ 792 _PREPARE(_access) }, \ 793 .guest_test = _access, \ 794 .mem_mark_cmd = _mark_cmd, \ 795 .guest_test_check = { _CHECK(_with_af) }, \ 796 .uffd_data_handler = _uffd_data_handler, \ 797 .uffd_pt_handler = _uffd_pt_handler, \ 798 .expected_events = { .uffd_faults = _uffd_faults, }, \ 799} 800 801#define TEST_DIRTY_LOG(_access, _with_af, _test_check, _pt_check) \ 802{ \ 803 .name = SCAT3(dirty_log, _access, _with_af), \ 804 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ 805 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ 806 .guest_prepare = { _PREPARE(_with_af), \ 807 _PREPARE(_access) }, \ 808 .guest_test = _access, \ 809 .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \ 810 .expected_events = { 0 }, \ 811} 812 813#define TEST_UFFD_AND_DIRTY_LOG(_access, _with_af, _uffd_data_handler, \ 814 _uffd_faults, _test_check, _pt_check) \ 815{ \ 816 .name = SCAT3(uffd_and_dirty_log, _access, _with_af), \ 817 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ 818 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ 819 .guest_prepare = { _PREPARE(_with_af), \ 820 _PREPARE(_access) }, \ 821 .guest_test = _access, \ 822 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ 823 .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \ 824 .uffd_data_handler = _uffd_data_handler, \ 825 .uffd_pt_handler = uffd_pt_handler, \ 826 .expected_events = { .uffd_faults = _uffd_faults, }, \ 827} 828 829#define TEST_RO_MEMSLOT(_access, _mmio_handler, _mmio_exits) \ 830{ \ 831 .name = SCAT2(ro_memslot, _access), \ 832 .data_memslot_flags = KVM_MEM_READONLY, \ 833 .pt_memslot_flags = KVM_MEM_READONLY, \ 834 .guest_prepare = { _PREPARE(_access) }, \ 835 .guest_test = _access, \ 836 .mmio_handler = _mmio_handler, \ 837 .expected_events = { .mmio_exits = _mmio_exits }, \ 838} 839 840#define TEST_RO_MEMSLOT_NO_SYNDROME(_access) \ 841{ \ 842 .name = SCAT2(ro_memslot_no_syndrome, _access), \ 843 .data_memslot_flags = KVM_MEM_READONLY, \ 844 .pt_memslot_flags = KVM_MEM_READONLY, \ 845 .guest_prepare = { _PREPARE(_access) }, \ 846 .guest_test = _access, \ 847 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ 848 .expected_events = { .fail_vcpu_runs = 1 }, \ 849} 850 851#define TEST_RO_MEMSLOT_AND_DIRTY_LOG(_access, _mmio_handler, _mmio_exits, \ 852 _test_check) \ 853{ \ 854 .name = SCAT2(ro_memslot, _access), \ 855 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ 856 .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ 857 .guest_prepare = { _PREPARE(_access) }, \ 858 .guest_test = _access, \ 859 .guest_test_check = { _test_check }, \ 860 .mmio_handler = _mmio_handler, \ 861 .expected_events = { .mmio_exits = _mmio_exits}, \ 862} 863 864#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(_access, _test_check) \ 865{ \ 866 .name = SCAT2(ro_memslot_no_syn_and_dlog, _access), \ 867 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ 868 .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ 869 .guest_prepare = { _PREPARE(_access) }, \ 870 .guest_test = _access, \ 871 .guest_test_check = { _test_check }, \ 872 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ 873 .expected_events = { .fail_vcpu_runs = 1 }, \ 874} 875 876#define TEST_RO_MEMSLOT_AND_UFFD(_access, _mmio_handler, _mmio_exits, \ 877 _uffd_data_handler, _uffd_faults) \ 878{ \ 879 .name = SCAT2(ro_memslot_uffd, _access), \ 880 .data_memslot_flags = KVM_MEM_READONLY, \ 881 .pt_memslot_flags = KVM_MEM_READONLY, \ 882 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ 883 .guest_prepare = { _PREPARE(_access) }, \ 884 .guest_test = _access, \ 885 .uffd_data_handler = _uffd_data_handler, \ 886 .uffd_pt_handler = uffd_pt_handler, \ 887 .mmio_handler = _mmio_handler, \ 888 .expected_events = { .mmio_exits = _mmio_exits, \ 889 .uffd_faults = _uffd_faults }, \ 890} 891 892#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(_access, _uffd_data_handler, \ 893 _uffd_faults) \ 894{ \ 895 .name = SCAT2(ro_memslot_no_syndrome, _access), \ 896 .data_memslot_flags = KVM_MEM_READONLY, \ 897 .pt_memslot_flags = KVM_MEM_READONLY, \ 898 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ 899 .guest_prepare = { _PREPARE(_access) }, \ 900 .guest_test = _access, \ 901 .uffd_data_handler = _uffd_data_handler, \ 902 .uffd_pt_handler = uffd_pt_handler, \ 903 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ 904 .expected_events = { .fail_vcpu_runs = 1, \ 905 .uffd_faults = _uffd_faults }, \ 906} 907 908static struct test_desc tests[] = { 909 910 /* Check that HW is setting the Access Flag (AF) (sanity checks). */ 911 TEST_ACCESS(guest_read64, with_af, CMD_NONE), 912 TEST_ACCESS(guest_ld_preidx, with_af, CMD_NONE), 913 TEST_ACCESS(guest_cas, with_af, CMD_NONE), 914 TEST_ACCESS(guest_write64, with_af, CMD_NONE), 915 TEST_ACCESS(guest_st_preidx, with_af, CMD_NONE), 916 TEST_ACCESS(guest_dc_zva, with_af, CMD_NONE), 917 TEST_ACCESS(guest_exec, with_af, CMD_NONE), 918 919 /* 920 * Punch a hole in the data backing store, and then try multiple 921 * accesses: reads should rturn zeroes, and writes should 922 * re-populate the page. Moreover, the test also check that no 923 * exception was generated in the guest. Note that this 924 * reading/writing behavior is the same as reading/writing a 925 * punched page (with fallocate(FALLOC_FL_PUNCH_HOLE)) from 926 * userspace. 927 */ 928 TEST_ACCESS(guest_read64, no_af, CMD_HOLE_DATA), 929 TEST_ACCESS(guest_cas, no_af, CMD_HOLE_DATA), 930 TEST_ACCESS(guest_ld_preidx, no_af, CMD_HOLE_DATA), 931 TEST_ACCESS(guest_write64, no_af, CMD_HOLE_DATA), 932 TEST_ACCESS(guest_st_preidx, no_af, CMD_HOLE_DATA), 933 TEST_ACCESS(guest_at, no_af, CMD_HOLE_DATA), 934 TEST_ACCESS(guest_dc_zva, no_af, CMD_HOLE_DATA), 935 936 /* 937 * Punch holes in the data and PT backing stores and mark them for 938 * userfaultfd handling. This should result in 2 faults: the access 939 * on the data backing store, and its respective S1 page table walk 940 * (S1PTW). 941 */ 942 TEST_UFFD(guest_read64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 943 uffd_data_handler, uffd_pt_handler, 2), 944 TEST_UFFD(guest_read64, no_af, CMD_HOLE_DATA | CMD_HOLE_PT, 945 uffd_data_handler, uffd_pt_handler, 2), 946 TEST_UFFD(guest_cas, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 947 uffd_data_handler, uffd_pt_handler, 2), 948 /* 949 * Can't test guest_at with_af as it's IMPDEF whether the AF is set. 950 * The S1PTW fault should still be marked as a write. 951 */ 952 TEST_UFFD(guest_at, no_af, CMD_HOLE_DATA | CMD_HOLE_PT, 953 uffd_no_handler, uffd_pt_handler, 1), 954 TEST_UFFD(guest_ld_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 955 uffd_data_handler, uffd_pt_handler, 2), 956 TEST_UFFD(guest_write64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 957 uffd_data_handler, uffd_pt_handler, 2), 958 TEST_UFFD(guest_dc_zva, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 959 uffd_data_handler, uffd_pt_handler, 2), 960 TEST_UFFD(guest_st_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 961 uffd_data_handler, uffd_pt_handler, 2), 962 TEST_UFFD(guest_exec, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, 963 uffd_data_handler, uffd_pt_handler, 2), 964 965 /* 966 * Try accesses when the data and PT memory regions are both 967 * tracked for dirty logging. 968 */ 969 TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log, 970 guest_check_s1ptw_wr_in_dirty_log), 971 TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log, 972 guest_check_no_s1ptw_wr_in_dirty_log), 973 TEST_DIRTY_LOG(guest_ld_preidx, with_af, 974 guest_check_no_write_in_dirty_log, 975 guest_check_s1ptw_wr_in_dirty_log), 976 TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log, 977 guest_check_no_s1ptw_wr_in_dirty_log), 978 TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log, 979 guest_check_s1ptw_wr_in_dirty_log), 980 TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log, 981 guest_check_s1ptw_wr_in_dirty_log), 982 TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log, 983 guest_check_s1ptw_wr_in_dirty_log), 984 TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log, 985 guest_check_s1ptw_wr_in_dirty_log), 986 TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log, 987 guest_check_s1ptw_wr_in_dirty_log), 988 989 /* 990 * Access when the data and PT memory regions are both marked for 991 * dirty logging and UFFD at the same time. The expected result is 992 * that writes should mark the dirty log and trigger a userfaultfd 993 * write fault. Reads/execs should result in a read userfaultfd 994 * fault, and nothing in the dirty log. Any S1PTW should result in 995 * a write in the dirty log and a userfaultfd write. 996 */ 997 TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af, 998 uffd_data_handler, 2, 999 guest_check_no_write_in_dirty_log, 1000 guest_check_s1ptw_wr_in_dirty_log), 1001 TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af, 1002 uffd_data_handler, 2, 1003 guest_check_no_write_in_dirty_log, 1004 guest_check_no_s1ptw_wr_in_dirty_log), 1005 TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af, 1006 uffd_data_handler, 1007 2, guest_check_no_write_in_dirty_log, 1008 guest_check_s1ptw_wr_in_dirty_log), 1009 TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, uffd_no_handler, 1, 1010 guest_check_no_write_in_dirty_log, 1011 guest_check_s1ptw_wr_in_dirty_log), 1012 TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af, 1013 uffd_data_handler, 2, 1014 guest_check_no_write_in_dirty_log, 1015 guest_check_s1ptw_wr_in_dirty_log), 1016 TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af, 1017 uffd_data_handler, 1018 2, guest_check_write_in_dirty_log, 1019 guest_check_s1ptw_wr_in_dirty_log), 1020 TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af, 1021 uffd_data_handler, 2, 1022 guest_check_write_in_dirty_log, 1023 guest_check_s1ptw_wr_in_dirty_log), 1024 TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af, 1025 uffd_data_handler, 1026 2, guest_check_write_in_dirty_log, 1027 guest_check_s1ptw_wr_in_dirty_log), 1028 TEST_UFFD_AND_DIRTY_LOG(guest_st_preidx, with_af, 1029 uffd_data_handler, 2, 1030 guest_check_write_in_dirty_log, 1031 guest_check_s1ptw_wr_in_dirty_log), 1032 /* 1033 * Access when both the PT and data regions are marked read-only 1034 * (with KVM_MEM_READONLY). Writes with a syndrome result in an 1035 * MMIO exit, writes with no syndrome (e.g., CAS) result in a 1036 * failed vcpu run, and reads/execs with and without syndroms do 1037 * not fault. 1038 */ 1039 TEST_RO_MEMSLOT(guest_read64, 0, 0), 1040 TEST_RO_MEMSLOT(guest_ld_preidx, 0, 0), 1041 TEST_RO_MEMSLOT(guest_at, 0, 0), 1042 TEST_RO_MEMSLOT(guest_exec, 0, 0), 1043 TEST_RO_MEMSLOT(guest_write64, mmio_on_test_gpa_handler, 1), 1044 TEST_RO_MEMSLOT_NO_SYNDROME(guest_dc_zva), 1045 TEST_RO_MEMSLOT_NO_SYNDROME(guest_cas), 1046 TEST_RO_MEMSLOT_NO_SYNDROME(guest_st_preidx), 1047 1048 /* 1049 * The PT and data regions are both read-only and marked 1050 * for dirty logging at the same time. The expected result is that 1051 * for writes there should be no write in the dirty log. The 1052 * readonly handling is the same as if the memslot was not marked 1053 * for dirty logging: writes with a syndrome result in an MMIO 1054 * exit, and writes with no syndrome result in a failed vcpu run. 1055 */ 1056 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_read64, 0, 0, 1057 guest_check_no_write_in_dirty_log), 1058 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_ld_preidx, 0, 0, 1059 guest_check_no_write_in_dirty_log), 1060 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_at, 0, 0, 1061 guest_check_no_write_in_dirty_log), 1062 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_exec, 0, 0, 1063 guest_check_no_write_in_dirty_log), 1064 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_write64, mmio_on_test_gpa_handler, 1065 1, guest_check_no_write_in_dirty_log), 1066 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_dc_zva, 1067 guest_check_no_write_in_dirty_log), 1068 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_cas, 1069 guest_check_no_write_in_dirty_log), 1070 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_st_preidx, 1071 guest_check_no_write_in_dirty_log), 1072 1073 /* 1074 * The PT and data regions are both read-only and punched with 1075 * holes tracked with userfaultfd. The expected result is the 1076 * union of both userfaultfd and read-only behaviors. For example, 1077 * write accesses result in a userfaultfd write fault and an MMIO 1078 * exit. Writes with no syndrome result in a failed vcpu run and 1079 * no userfaultfd write fault. Reads result in userfaultfd getting 1080 * triggered. 1081 */ 1082 TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0, uffd_data_handler, 2), 1083 TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0, uffd_data_handler, 2), 1084 TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0, uffd_no_handler, 1), 1085 TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0, uffd_data_handler, 2), 1086 TEST_RO_MEMSLOT_AND_UFFD(guest_write64, mmio_on_test_gpa_handler, 1, 1087 uffd_data_handler, 2), 1088 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas, uffd_data_handler, 2), 1089 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva, uffd_no_handler, 1), 1090 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx, uffd_no_handler, 1), 1091 1092 { 0 } 1093}; 1094 1095static void for_each_test_and_guest_mode(enum vm_mem_backing_src_type src_type) 1096{ 1097 struct test_desc *t; 1098 1099 for (t = &tests[0]; t->name; t++) { 1100 if (t->skip) 1101 continue; 1102 1103 struct test_params p = { 1104 .src_type = src_type, 1105 .test_desc = t, 1106 }; 1107 1108 for_each_guest_mode(run_test, &p); 1109 } 1110} 1111 1112int main(int argc, char *argv[]) 1113{ 1114 enum vm_mem_backing_src_type src_type; 1115 int opt; 1116 1117 src_type = DEFAULT_VM_MEM_SRC; 1118 1119 while ((opt = getopt(argc, argv, "hm:s:")) != -1) { 1120 switch (opt) { 1121 case 'm': 1122 guest_modes_cmdline(optarg); 1123 break; 1124 case 's': 1125 src_type = parse_backing_src_type(optarg); 1126 break; 1127 case 'h': 1128 default: 1129 help(argv[0]); 1130 exit(0); 1131 } 1132 } 1133 1134 for_each_test_and_guest_mode(src_type); 1135 return 0; 1136} 1137