1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Userfaultfd unit tests. 4 * 5 * Copyright (C) 2015-2023 Red Hat, Inc. 6 */ 7 8#include "uffd-common.h" 9 10#include "../../../../mm/gup_test.h" 11 12#ifdef __NR_userfaultfd 13 14/* The unit test doesn't need a large or random size, make it 32MB for now */ 15#define UFFD_TEST_MEM_SIZE (32UL << 20) 16 17#define MEM_ANON BIT_ULL(0) 18#define MEM_SHMEM BIT_ULL(1) 19#define MEM_SHMEM_PRIVATE BIT_ULL(2) 20#define MEM_HUGETLB BIT_ULL(3) 21#define MEM_HUGETLB_PRIVATE BIT_ULL(4) 22 23#define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \ 24 MEM_HUGETLB | MEM_HUGETLB_PRIVATE) 25 26#define ALIGN_UP(x, align_to) \ 27 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1))) 28 29struct mem_type { 30 const char *name; 31 unsigned int mem_flag; 32 uffd_test_ops_t *mem_ops; 33 bool shared; 34}; 35typedef struct mem_type mem_type_t; 36 37mem_type_t mem_types[] = { 38 { 39 .name = "anon", 40 .mem_flag = MEM_ANON, 41 .mem_ops = &anon_uffd_test_ops, 42 .shared = false, 43 }, 44 { 45 .name = "shmem", 46 .mem_flag = MEM_SHMEM, 47 .mem_ops = &shmem_uffd_test_ops, 48 .shared = true, 49 }, 50 { 51 .name = "shmem-private", 52 .mem_flag = MEM_SHMEM_PRIVATE, 53 .mem_ops = &shmem_uffd_test_ops, 54 .shared = false, 55 }, 56 { 57 .name = "hugetlb", 58 .mem_flag = MEM_HUGETLB, 59 .mem_ops = &hugetlb_uffd_test_ops, 60 .shared = true, 61 }, 62 { 63 .name = "hugetlb-private", 64 .mem_flag = MEM_HUGETLB_PRIVATE, 65 .mem_ops = &hugetlb_uffd_test_ops, 66 .shared = false, 67 }, 68}; 69 70/* Arguments to be passed over to each uffd unit test */ 71struct uffd_test_args { 72 mem_type_t *mem_type; 73}; 74typedef struct uffd_test_args uffd_test_args_t; 75 76/* Returns: UFFD_TEST_* */ 77typedef void (*uffd_test_fn)(uffd_test_args_t *); 78 79typedef struct { 80 const char *name; 81 uffd_test_fn uffd_fn; 82 unsigned int mem_targets; 83 uint64_t uffd_feature_required; 84 uffd_test_case_ops_t *test_case_ops; 85} uffd_test_case_t; 86 87static void uffd_test_report(void) 88{ 89 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n", 90 ksft_get_pass_cnt(), 91 ksft_get_xskip_cnt(), 92 ksft_get_fail_cnt(), 93 ksft_test_num()); 94} 95 96static void uffd_test_pass(void) 97{ 98 printf("done\n"); 99 ksft_inc_pass_cnt(); 100} 101 102#define uffd_test_start(...) do { \ 103 printf("Testing "); \ 104 printf(__VA_ARGS__); \ 105 printf("... "); \ 106 fflush(stdout); \ 107 } while (0) 108 109#define uffd_test_fail(...) do { \ 110 printf("failed [reason: "); \ 111 printf(__VA_ARGS__); \ 112 printf("]\n"); \ 113 ksft_inc_fail_cnt(); \ 114 } while (0) 115 116static void uffd_test_skip(const char *message) 117{ 118 printf("skipped [reason: %s]\n", message); 119 ksft_inc_xskip_cnt(); 120} 121 122/* 123 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll 124 * return 1 even if some test failed as long as uffd supported, because in 125 * that case we still want to proceed with the rest uffd unit tests. 126 */ 127static int test_uffd_api(bool use_dev) 128{ 129 struct uffdio_api uffdio_api; 130 int uffd; 131 132 uffd_test_start("UFFDIO_API (with %s)", 133 use_dev ? "/dev/userfaultfd" : "syscall"); 134 135 if (use_dev) 136 uffd = uffd_open_dev(UFFD_FLAGS); 137 else 138 uffd = uffd_open_sys(UFFD_FLAGS); 139 if (uffd < 0) { 140 uffd_test_skip("cannot open userfaultfd handle"); 141 return 0; 142 } 143 144 /* Test wrong UFFD_API */ 145 uffdio_api.api = 0xab; 146 uffdio_api.features = 0; 147 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { 148 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't"); 149 goto out; 150 } 151 152 /* Test wrong feature bit */ 153 uffdio_api.api = UFFD_API; 154 uffdio_api.features = BIT_ULL(63); 155 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { 156 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't"); 157 goto out; 158 } 159 160 /* Test normal UFFDIO_API */ 161 uffdio_api.api = UFFD_API; 162 uffdio_api.features = 0; 163 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) { 164 uffd_test_fail("UFFDIO_API should succeed but failed"); 165 goto out; 166 } 167 168 /* Test double requests of UFFDIO_API with a random feature set */ 169 uffdio_api.features = BIT_ULL(0); 170 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { 171 uffd_test_fail("UFFDIO_API should reject initialized uffd"); 172 goto out; 173 } 174 175 uffd_test_pass(); 176out: 177 close(uffd); 178 /* We have a valid uffd handle */ 179 return 1; 180} 181 182/* 183 * This function initializes the global variables. TODO: remove global 184 * vars and then remove this. 185 */ 186static int 187uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test, 188 mem_type_t *mem_type, const char **errmsg) 189{ 190 map_shared = mem_type->shared; 191 uffd_test_ops = mem_type->mem_ops; 192 uffd_test_case_ops = test->test_case_ops; 193 194 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) 195 page_size = default_huge_page_size(); 196 else 197 page_size = psize(); 198 199 nr_pages = UFFD_TEST_MEM_SIZE / page_size; 200 /* TODO: remove this global var.. it's so ugly */ 201 nr_cpus = 1; 202 203 /* Initialize test arguments */ 204 args->mem_type = mem_type; 205 206 return uffd_test_ctx_init(test->uffd_feature_required, errmsg); 207} 208 209static bool uffd_feature_supported(uffd_test_case_t *test) 210{ 211 uint64_t features; 212 213 if (uffd_get_features(&features)) 214 return false; 215 216 return (features & test->uffd_feature_required) == 217 test->uffd_feature_required; 218} 219 220static int pagemap_open(void) 221{ 222 int fd = open("/proc/self/pagemap", O_RDONLY); 223 224 if (fd < 0) 225 err("open pagemap"); 226 227 return fd; 228} 229 230/* This macro let __LINE__ works in err() */ 231#define pagemap_check_wp(value, wp) do { \ 232 if (!!(value & PM_UFFD_WP) != wp) \ 233 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \ 234 } while (0) 235 236typedef struct { 237 int parent_uffd, child_uffd; 238} fork_event_args; 239 240static void *fork_event_consumer(void *data) 241{ 242 fork_event_args *args = data; 243 struct uffd_msg msg = { 0 }; 244 245 /* Read until a full msg received */ 246 while (uffd_read_msg(args->parent_uffd, &msg)); 247 248 if (msg.event != UFFD_EVENT_FORK) 249 err("wrong message: %u\n", msg.event); 250 251 /* Just to be properly freed later */ 252 args->child_uffd = msg.arg.fork.ufd; 253 return NULL; 254} 255 256typedef struct { 257 int gup_fd; 258 bool pinned; 259} pin_args; 260 261/* 262 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired 263 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy 264 * all needs of the test cases (e.g., trigger unshare, trigger fork() early 265 * CoW, etc.). 266 */ 267static int pin_pages(pin_args *args, void *buffer, size_t size) 268{ 269 struct pin_longterm_test test = { 270 .addr = (uintptr_t)buffer, 271 .size = size, 272 /* Read-only pins */ 273 .flags = 0, 274 }; 275 276 if (args->pinned) 277 err("already pinned"); 278 279 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); 280 if (args->gup_fd < 0) 281 return -errno; 282 283 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) { 284 /* Even if gup_test existed, can be an old gup_test / kernel */ 285 close(args->gup_fd); 286 return -errno; 287 } 288 args->pinned = true; 289 return 0; 290} 291 292static void unpin_pages(pin_args *args) 293{ 294 if (!args->pinned) 295 err("unpin without pin first"); 296 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP)) 297 err("PIN_LONGTERM_TEST_STOP"); 298 close(args->gup_fd); 299 args->pinned = false; 300} 301 302static int pagemap_test_fork(int uffd, bool with_event, bool test_pin) 303{ 304 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 }; 305 pthread_t thread; 306 pid_t child; 307 uint64_t value; 308 int fd, result; 309 310 /* Prepare a thread to resolve EVENT_FORK */ 311 if (with_event) { 312 if (pthread_create(&thread, NULL, fork_event_consumer, &args)) 313 err("pthread_create()"); 314 } 315 316 child = fork(); 317 if (!child) { 318 /* Open the pagemap fd of the child itself */ 319 pin_args args = {}; 320 321 fd = pagemap_open(); 322 323 if (test_pin && pin_pages(&args, area_dst, page_size)) 324 /* 325 * Normally when reach here we have pinned in 326 * previous tests, so shouldn't fail anymore 327 */ 328 err("pin page failed in child"); 329 330 value = pagemap_get_entry(fd, area_dst); 331 /* 332 * After fork(), we should handle uffd-wp bit differently: 333 * 334 * (1) when with EVENT_FORK, it should persist 335 * (2) when without EVENT_FORK, it should be dropped 336 */ 337 pagemap_check_wp(value, with_event); 338 if (test_pin) 339 unpin_pages(&args); 340 /* Succeed */ 341 exit(0); 342 } 343 waitpid(child, &result, 0); 344 345 if (with_event) { 346 if (pthread_join(thread, NULL)) 347 err("pthread_join()"); 348 if (args.child_uffd < 0) 349 err("Didn't receive child uffd"); 350 close(args.child_uffd); 351 } 352 353 return result; 354} 355 356static void uffd_wp_unpopulated_test(uffd_test_args_t *args) 357{ 358 uint64_t value; 359 int pagemap_fd; 360 361 if (uffd_register(uffd, area_dst, nr_pages * page_size, 362 false, true, false)) 363 err("register failed"); 364 365 pagemap_fd = pagemap_open(); 366 367 /* Test applying pte marker to anon unpopulated */ 368 wp_range(uffd, (uint64_t)area_dst, page_size, true); 369 value = pagemap_get_entry(pagemap_fd, area_dst); 370 pagemap_check_wp(value, true); 371 372 /* Test unprotect on anon pte marker */ 373 wp_range(uffd, (uint64_t)area_dst, page_size, false); 374 value = pagemap_get_entry(pagemap_fd, area_dst); 375 pagemap_check_wp(value, false); 376 377 /* Test zap on anon marker */ 378 wp_range(uffd, (uint64_t)area_dst, page_size, true); 379 if (madvise(area_dst, page_size, MADV_DONTNEED)) 380 err("madvise(MADV_DONTNEED) failed"); 381 value = pagemap_get_entry(pagemap_fd, area_dst); 382 pagemap_check_wp(value, false); 383 384 /* Test fault in after marker removed */ 385 *area_dst = 1; 386 value = pagemap_get_entry(pagemap_fd, area_dst); 387 pagemap_check_wp(value, false); 388 /* Drop it to make pte none again */ 389 if (madvise(area_dst, page_size, MADV_DONTNEED)) 390 err("madvise(MADV_DONTNEED) failed"); 391 392 /* Test read-zero-page upon pte marker */ 393 wp_range(uffd, (uint64_t)area_dst, page_size, true); 394 *(volatile char *)area_dst; 395 /* Drop it to make pte none again */ 396 if (madvise(area_dst, page_size, MADV_DONTNEED)) 397 err("madvise(MADV_DONTNEED) failed"); 398 399 uffd_test_pass(); 400} 401 402static void uffd_wp_fork_test_common(uffd_test_args_t *args, 403 bool with_event) 404{ 405 int pagemap_fd; 406 uint64_t value; 407 408 if (uffd_register(uffd, area_dst, nr_pages * page_size, 409 false, true, false)) 410 err("register failed"); 411 412 pagemap_fd = pagemap_open(); 413 414 /* Touch the page */ 415 *area_dst = 1; 416 wp_range(uffd, (uint64_t)area_dst, page_size, true); 417 value = pagemap_get_entry(pagemap_fd, area_dst); 418 pagemap_check_wp(value, true); 419 if (pagemap_test_fork(uffd, with_event, false)) { 420 uffd_test_fail("Detected %s uffd-wp bit in child in present pte", 421 with_event ? "missing" : "stall"); 422 goto out; 423 } 424 425 /* 426 * This is an attempt for zapping the pgtable so as to test the 427 * markers. 428 * 429 * For private mappings, PAGEOUT will only work on exclusive ptes 430 * (PM_MMAP_EXCLUSIVE) which we should satisfy. 431 * 432 * For shared, PAGEOUT may not work. Use DONTNEED instead which 433 * plays a similar role of zapping (rather than freeing the page) 434 * to expose pte markers. 435 */ 436 if (args->mem_type->shared) { 437 if (madvise(area_dst, page_size, MADV_DONTNEED)) 438 err("MADV_DONTNEED"); 439 } else { 440 /* 441 * NOTE: ignore retval because private-hugetlb doesn't yet 442 * support swapping, so it could fail. 443 */ 444 madvise(area_dst, page_size, MADV_PAGEOUT); 445 } 446 447 /* Uffd-wp should persist even swapped out */ 448 value = pagemap_get_entry(pagemap_fd, area_dst); 449 pagemap_check_wp(value, true); 450 if (pagemap_test_fork(uffd, with_event, false)) { 451 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte", 452 with_event ? "missing" : "stall"); 453 goto out; 454 } 455 456 /* Unprotect; this tests swap pte modifications */ 457 wp_range(uffd, (uint64_t)area_dst, page_size, false); 458 value = pagemap_get_entry(pagemap_fd, area_dst); 459 pagemap_check_wp(value, false); 460 461 /* Fault in the page from disk */ 462 *area_dst = 2; 463 value = pagemap_get_entry(pagemap_fd, area_dst); 464 pagemap_check_wp(value, false); 465 uffd_test_pass(); 466out: 467 if (uffd_unregister(uffd, area_dst, nr_pages * page_size)) 468 err("unregister failed"); 469 close(pagemap_fd); 470} 471 472static void uffd_wp_fork_test(uffd_test_args_t *args) 473{ 474 uffd_wp_fork_test_common(args, false); 475} 476 477static void uffd_wp_fork_with_event_test(uffd_test_args_t *args) 478{ 479 uffd_wp_fork_test_common(args, true); 480} 481 482static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args, 483 bool with_event) 484{ 485 int pagemap_fd; 486 pin_args pin_args = {}; 487 488 if (uffd_register(uffd, area_dst, page_size, false, true, false)) 489 err("register failed"); 490 491 pagemap_fd = pagemap_open(); 492 493 /* Touch the page */ 494 *area_dst = 1; 495 wp_range(uffd, (uint64_t)area_dst, page_size, true); 496 497 /* 498 * 1. First pin, then fork(). This tests fork() special path when 499 * doing early CoW if the page is private. 500 */ 501 if (pin_pages(&pin_args, area_dst, page_size)) { 502 uffd_test_skip("Possibly CONFIG_GUP_TEST missing " 503 "or unprivileged"); 504 close(pagemap_fd); 505 uffd_unregister(uffd, area_dst, page_size); 506 return; 507 } 508 509 if (pagemap_test_fork(uffd, with_event, false)) { 510 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()", 511 with_event ? "missing" : "stall"); 512 unpin_pages(&pin_args); 513 goto out; 514 } 515 516 unpin_pages(&pin_args); 517 518 /* 519 * 2. First fork(), then pin (in the child, where test_pin==true). 520 * This tests COR, aka, page unsharing on private memories. 521 */ 522 if (pagemap_test_fork(uffd, with_event, true)) { 523 uffd_test_fail("Detected %s uffd-wp bit when RO pin", 524 with_event ? "missing" : "stall"); 525 goto out; 526 } 527 uffd_test_pass(); 528out: 529 if (uffd_unregister(uffd, area_dst, page_size)) 530 err("register failed"); 531 close(pagemap_fd); 532} 533 534static void uffd_wp_fork_pin_test(uffd_test_args_t *args) 535{ 536 uffd_wp_fork_pin_test_common(args, false); 537} 538 539static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args) 540{ 541 uffd_wp_fork_pin_test_common(args, true); 542} 543 544static void check_memory_contents(char *p) 545{ 546 unsigned long i, j; 547 uint8_t expected_byte; 548 549 for (i = 0; i < nr_pages; ++i) { 550 expected_byte = ~((uint8_t)(i % ((uint8_t)-1))); 551 for (j = 0; j < page_size; j++) { 552 uint8_t v = *(uint8_t *)(p + (i * page_size) + j); 553 if (v != expected_byte) 554 err("unexpected page contents"); 555 } 556 } 557} 558 559static void uffd_minor_test_common(bool test_collapse, bool test_wp) 560{ 561 unsigned long p; 562 pthread_t uffd_mon; 563 char c; 564 struct uffd_args args = { 0 }; 565 566 /* 567 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing 568 * both do not make much sense. 569 */ 570 assert(!(test_collapse && test_wp)); 571 572 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size, 573 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */ 574 false, test_wp, true)) 575 err("register failure"); 576 577 /* 578 * After registering with UFFD, populate the non-UFFD-registered side of 579 * the shared mapping. This should *not* trigger any UFFD minor faults. 580 */ 581 for (p = 0; p < nr_pages; ++p) 582 memset(area_dst + (p * page_size), p % ((uint8_t)-1), 583 page_size); 584 585 args.apply_wp = test_wp; 586 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 587 err("uffd_poll_thread create"); 588 589 /* 590 * Read each of the pages back using the UFFD-registered mapping. We 591 * expect that the first time we touch a page, it will result in a minor 592 * fault. uffd_poll_thread will resolve the fault by bit-flipping the 593 * page's contents, and then issuing a CONTINUE ioctl. 594 */ 595 check_memory_contents(area_dst_alias); 596 597 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 598 err("pipe write"); 599 if (pthread_join(uffd_mon, NULL)) 600 err("join() failed"); 601 602 if (test_collapse) { 603 if (madvise(area_dst_alias, nr_pages * page_size, 604 MADV_COLLAPSE)) { 605 /* It's fine to fail for this one... */ 606 uffd_test_skip("MADV_COLLAPSE failed"); 607 return; 608 } 609 610 uffd_test_ops->check_pmd_mapping(area_dst, 611 nr_pages * page_size / 612 read_pmd_pagesize()); 613 /* 614 * This won't cause uffd-fault - it purely just makes sure there 615 * was no corruption. 616 */ 617 check_memory_contents(area_dst_alias); 618 } 619 620 if (args.missing_faults != 0 || args.minor_faults != nr_pages) 621 uffd_test_fail("stats check error"); 622 else 623 uffd_test_pass(); 624} 625 626void uffd_minor_test(uffd_test_args_t *args) 627{ 628 uffd_minor_test_common(false, false); 629} 630 631void uffd_minor_wp_test(uffd_test_args_t *args) 632{ 633 uffd_minor_test_common(false, true); 634} 635 636void uffd_minor_collapse_test(uffd_test_args_t *args) 637{ 638 uffd_minor_test_common(true, false); 639} 640 641static sigjmp_buf jbuf, *sigbuf; 642 643static void sighndl(int sig, siginfo_t *siginfo, void *ptr) 644{ 645 if (sig == SIGBUS) { 646 if (sigbuf) 647 siglongjmp(*sigbuf, 1); 648 abort(); 649 } 650} 651 652/* 653 * For non-cooperative userfaultfd test we fork() a process that will 654 * generate pagefaults, will mremap the area monitored by the 655 * userfaultfd and at last this process will release the monitored 656 * area. 657 * For the anonymous and shared memory the area is divided into two 658 * parts, the first part is accessed before mremap, and the second 659 * part is accessed after mremap. Since hugetlbfs does not support 660 * mremap, the entire monitored area is accessed in a single pass for 661 * HUGETLB_TEST. 662 * The release of the pages currently generates event for shmem and 663 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked 664 * for hugetlb. 665 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register 666 * monitored area, generate pagefaults and test that signal is delivered. 667 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2 668 * test robustness use case - we release monitored area, fork a process 669 * that will generate pagefaults and verify signal is generated. 670 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal 671 * feature. Using monitor thread, verify no userfault events are generated. 672 */ 673static int faulting_process(int signal_test, bool wp) 674{ 675 unsigned long nr, i; 676 unsigned long long count; 677 unsigned long split_nr_pages; 678 unsigned long lastnr; 679 struct sigaction act; 680 volatile unsigned long signalled = 0; 681 682 split_nr_pages = (nr_pages + 1) / 2; 683 684 if (signal_test) { 685 sigbuf = &jbuf; 686 memset(&act, 0, sizeof(act)); 687 act.sa_sigaction = sighndl; 688 act.sa_flags = SA_SIGINFO; 689 if (sigaction(SIGBUS, &act, 0)) 690 err("sigaction"); 691 lastnr = (unsigned long)-1; 692 } 693 694 for (nr = 0; nr < split_nr_pages; nr++) { 695 volatile int steps = 1; 696 unsigned long offset = nr * page_size; 697 698 if (signal_test) { 699 if (sigsetjmp(*sigbuf, 1) != 0) { 700 if (steps == 1 && nr == lastnr) 701 err("Signal repeated"); 702 703 lastnr = nr; 704 if (signal_test == 1) { 705 if (steps == 1) { 706 /* This is a MISSING request */ 707 steps++; 708 if (copy_page(uffd, offset, wp)) 709 signalled++; 710 } else { 711 /* This is a WP request */ 712 assert(steps == 2); 713 wp_range(uffd, 714 (__u64)area_dst + 715 offset, 716 page_size, false); 717 } 718 } else { 719 signalled++; 720 continue; 721 } 722 } 723 } 724 725 count = *area_count(area_dst, nr); 726 if (count != count_verify[nr]) 727 err("nr %lu memory corruption %llu %llu\n", 728 nr, count, count_verify[nr]); 729 /* 730 * Trigger write protection if there is by writing 731 * the same value back. 732 */ 733 *area_count(area_dst, nr) = count; 734 } 735 736 if (signal_test) 737 return signalled != split_nr_pages; 738 739 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size, 740 MREMAP_MAYMOVE | MREMAP_FIXED, area_src); 741 if (area_dst == MAP_FAILED) 742 err("mremap"); 743 /* Reset area_src since we just clobbered it */ 744 area_src = NULL; 745 746 for (; nr < nr_pages; nr++) { 747 count = *area_count(area_dst, nr); 748 if (count != count_verify[nr]) { 749 err("nr %lu memory corruption %llu %llu\n", 750 nr, count, count_verify[nr]); 751 } 752 /* 753 * Trigger write protection if there is by writing 754 * the same value back. 755 */ 756 *area_count(area_dst, nr) = count; 757 } 758 759 uffd_test_ops->release_pages(area_dst); 760 761 for (nr = 0; nr < nr_pages; nr++) 762 for (i = 0; i < page_size; i++) 763 if (*(area_dst + nr * page_size + i) != 0) 764 err("page %lu offset %lu is not zero", nr, i); 765 766 return 0; 767} 768 769static void uffd_sigbus_test_common(bool wp) 770{ 771 unsigned long userfaults; 772 pthread_t uffd_mon; 773 pid_t pid; 774 int err; 775 char c; 776 struct uffd_args args = { 0 }; 777 778 ready_for_fork = false; 779 780 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 781 782 if (uffd_register(uffd, area_dst, nr_pages * page_size, 783 true, wp, false)) 784 err("register failure"); 785 786 if (faulting_process(1, wp)) 787 err("faulting process failed"); 788 789 uffd_test_ops->release_pages(area_dst); 790 791 args.apply_wp = wp; 792 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 793 err("uffd_poll_thread create"); 794 795 while (!ready_for_fork) 796 ; /* Wait for the poll_thread to start executing before forking */ 797 798 pid = fork(); 799 if (pid < 0) 800 err("fork"); 801 802 if (!pid) 803 exit(faulting_process(2, wp)); 804 805 waitpid(pid, &err, 0); 806 if (err) 807 err("faulting process failed"); 808 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 809 err("pipe write"); 810 if (pthread_join(uffd_mon, (void **)&userfaults)) 811 err("pthread_join()"); 812 813 if (userfaults) 814 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults); 815 else 816 uffd_test_pass(); 817} 818 819static void uffd_sigbus_test(uffd_test_args_t *args) 820{ 821 uffd_sigbus_test_common(false); 822} 823 824static void uffd_sigbus_wp_test(uffd_test_args_t *args) 825{ 826 uffd_sigbus_test_common(true); 827} 828 829static void uffd_events_test_common(bool wp) 830{ 831 pthread_t uffd_mon; 832 pid_t pid; 833 int err; 834 char c; 835 struct uffd_args args = { 0 }; 836 837 ready_for_fork = false; 838 839 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 840 if (uffd_register(uffd, area_dst, nr_pages * page_size, 841 true, wp, false)) 842 err("register failure"); 843 844 args.apply_wp = wp; 845 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 846 err("uffd_poll_thread create"); 847 848 while (!ready_for_fork) 849 ; /* Wait for the poll_thread to start executing before forking */ 850 851 pid = fork(); 852 if (pid < 0) 853 err("fork"); 854 855 if (!pid) 856 exit(faulting_process(0, wp)); 857 858 waitpid(pid, &err, 0); 859 if (err) 860 err("faulting process failed"); 861 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 862 err("pipe write"); 863 if (pthread_join(uffd_mon, NULL)) 864 err("pthread_join()"); 865 866 if (args.missing_faults != nr_pages) 867 uffd_test_fail("Fault counts wrong"); 868 else 869 uffd_test_pass(); 870} 871 872static void uffd_events_test(uffd_test_args_t *args) 873{ 874 uffd_events_test_common(false); 875} 876 877static void uffd_events_wp_test(uffd_test_args_t *args) 878{ 879 uffd_events_test_common(true); 880} 881 882static void retry_uffdio_zeropage(int ufd, 883 struct uffdio_zeropage *uffdio_zeropage) 884{ 885 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start, 886 uffdio_zeropage->range.len, 887 0); 888 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) { 889 if (uffdio_zeropage->zeropage != -EEXIST) 890 err("UFFDIO_ZEROPAGE error: %"PRId64, 891 (int64_t)uffdio_zeropage->zeropage); 892 } else { 893 err("UFFDIO_ZEROPAGE error: %"PRId64, 894 (int64_t)uffdio_zeropage->zeropage); 895 } 896} 897 898static bool do_uffdio_zeropage(int ufd, bool has_zeropage) 899{ 900 struct uffdio_zeropage uffdio_zeropage = { 0 }; 901 int ret; 902 __s64 res; 903 904 uffdio_zeropage.range.start = (unsigned long) area_dst; 905 uffdio_zeropage.range.len = page_size; 906 uffdio_zeropage.mode = 0; 907 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage); 908 res = uffdio_zeropage.zeropage; 909 if (ret) { 910 /* real retval in ufdio_zeropage.zeropage */ 911 if (has_zeropage) 912 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res); 913 else if (res != -EINVAL) 914 err("UFFDIO_ZEROPAGE not -EINVAL"); 915 } else if (has_zeropage) { 916 if (res != page_size) 917 err("UFFDIO_ZEROPAGE unexpected size"); 918 else 919 retry_uffdio_zeropage(ufd, &uffdio_zeropage); 920 return true; 921 } else 922 err("UFFDIO_ZEROPAGE succeeded"); 923 924 return false; 925} 926 927/* 928 * Registers a range with MISSING mode only for zeropage test. Return true 929 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register() 930 * because we want to detect .ioctls along the way. 931 */ 932static bool 933uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len) 934{ 935 uint64_t ioctls = 0; 936 937 if (uffd_register_with_ioctls(uffd, addr, len, true, 938 false, false, &ioctls)) 939 err("zeropage register fail"); 940 941 return ioctls & (1 << _UFFDIO_ZEROPAGE); 942} 943 944/* exercise UFFDIO_ZEROPAGE */ 945static void uffd_zeropage_test(uffd_test_args_t *args) 946{ 947 bool has_zeropage; 948 int i; 949 950 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size); 951 if (area_dst_alias) 952 /* Ignore the retval; we already have it */ 953 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size); 954 955 if (do_uffdio_zeropage(uffd, has_zeropage)) 956 for (i = 0; i < page_size; i++) 957 if (area_dst[i] != 0) 958 err("data non-zero at offset %d\n", i); 959 960 if (uffd_unregister(uffd, area_dst, page_size)) 961 err("unregister"); 962 963 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size)) 964 err("unregister"); 965 966 uffd_test_pass(); 967} 968 969static void uffd_register_poison(int uffd, void *addr, uint64_t len) 970{ 971 uint64_t ioctls = 0; 972 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON); 973 974 if (uffd_register_with_ioctls(uffd, addr, len, true, 975 false, false, &ioctls)) 976 err("poison register fail"); 977 978 if ((ioctls & expected) != expected) 979 err("registered area doesn't support COPY and POISON ioctls"); 980} 981 982static void do_uffdio_poison(int uffd, unsigned long offset) 983{ 984 struct uffdio_poison uffdio_poison = { 0 }; 985 int ret; 986 __s64 res; 987 988 uffdio_poison.range.start = (unsigned long) area_dst + offset; 989 uffdio_poison.range.len = page_size; 990 uffdio_poison.mode = 0; 991 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison); 992 res = uffdio_poison.updated; 993 994 if (ret) 995 err("UFFDIO_POISON error: %"PRId64, (int64_t)res); 996 else if (res != page_size) 997 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res); 998} 999 1000static void uffd_poison_handle_fault( 1001 struct uffd_msg *msg, struct uffd_args *args) 1002{ 1003 unsigned long offset; 1004 1005 if (msg->event != UFFD_EVENT_PAGEFAULT) 1006 err("unexpected msg event %u", msg->event); 1007 1008 if (msg->arg.pagefault.flags & 1009 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR)) 1010 err("unexpected fault type %llu", msg->arg.pagefault.flags); 1011 1012 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; 1013 offset &= ~(page_size-1); 1014 1015 /* Odd pages -> copy zeroed page; even pages -> poison. */ 1016 if (offset & page_size) 1017 copy_page(uffd, offset, false); 1018 else 1019 do_uffdio_poison(uffd, offset); 1020} 1021 1022static void uffd_poison_test(uffd_test_args_t *targs) 1023{ 1024 pthread_t uffd_mon; 1025 char c; 1026 struct uffd_args args = { 0 }; 1027 struct sigaction act = { 0 }; 1028 unsigned long nr_sigbus = 0; 1029 unsigned long nr; 1030 1031 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 1032 1033 uffd_register_poison(uffd, area_dst, nr_pages * page_size); 1034 memset(area_src, 0, nr_pages * page_size); 1035 1036 args.handle_fault = uffd_poison_handle_fault; 1037 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 1038 err("uffd_poll_thread create"); 1039 1040 sigbuf = &jbuf; 1041 act.sa_sigaction = sighndl; 1042 act.sa_flags = SA_SIGINFO; 1043 if (sigaction(SIGBUS, &act, 0)) 1044 err("sigaction"); 1045 1046 for (nr = 0; nr < nr_pages; ++nr) { 1047 unsigned long offset = nr * page_size; 1048 const char *bytes = (const char *) area_dst + offset; 1049 const char *i; 1050 1051 if (sigsetjmp(*sigbuf, 1)) { 1052 /* 1053 * Access below triggered a SIGBUS, which was caught by 1054 * sighndl, which then jumped here. Count this SIGBUS, 1055 * and move on to next page. 1056 */ 1057 ++nr_sigbus; 1058 continue; 1059 } 1060 1061 for (i = bytes; i < bytes + page_size; ++i) { 1062 if (*i) 1063 err("nonzero byte in area_dst (%p) at %p: %u", 1064 area_dst, i, *i); 1065 } 1066 } 1067 1068 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 1069 err("pipe write"); 1070 if (pthread_join(uffd_mon, NULL)) 1071 err("pthread_join()"); 1072 1073 if (nr_sigbus != nr_pages / 2) 1074 err("expected to receive %lu SIGBUS, actually received %lu", 1075 nr_pages / 2, nr_sigbus); 1076 1077 uffd_test_pass(); 1078} 1079 1080static void 1081uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args, 1082 unsigned long len) 1083{ 1084 unsigned long offset; 1085 1086 if (msg->event != UFFD_EVENT_PAGEFAULT) 1087 err("unexpected msg event %u", msg->event); 1088 1089 if (msg->arg.pagefault.flags & 1090 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE)) 1091 err("unexpected fault type %llu", msg->arg.pagefault.flags); 1092 1093 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; 1094 offset &= ~(len-1); 1095 1096 if (move_page(uffd, offset, len)) 1097 args->missing_faults++; 1098} 1099 1100static void uffd_move_handle_fault(struct uffd_msg *msg, 1101 struct uffd_args *args) 1102{ 1103 uffd_move_handle_fault_common(msg, args, page_size); 1104} 1105 1106static void uffd_move_pmd_handle_fault(struct uffd_msg *msg, 1107 struct uffd_args *args) 1108{ 1109 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize()); 1110} 1111 1112static void 1113uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size, 1114 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args)) 1115{ 1116 unsigned long nr; 1117 pthread_t uffd_mon; 1118 char c; 1119 unsigned long long count; 1120 struct uffd_args args = { 0 }; 1121 char *orig_area_src, *orig_area_dst; 1122 unsigned long step_size, step_count; 1123 unsigned long src_offs = 0; 1124 unsigned long dst_offs = 0; 1125 1126 /* Prevent source pages from being mapped more than once */ 1127 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK)) 1128 err("madvise(MADV_DONTFORK) failure"); 1129 1130 if (uffd_register(uffd, area_dst, nr_pages * page_size, 1131 true, false, false)) 1132 err("register failure"); 1133 1134 args.handle_fault = handle_fault; 1135 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 1136 err("uffd_poll_thread create"); 1137 1138 step_size = chunk_size / page_size; 1139 step_count = nr_pages / step_size; 1140 1141 if (chunk_size > page_size) { 1142 char *aligned_src = ALIGN_UP(area_src, chunk_size); 1143 char *aligned_dst = ALIGN_UP(area_dst, chunk_size); 1144 1145 if (aligned_src != area_src || aligned_dst != area_dst) { 1146 src_offs = (aligned_src - area_src) / page_size; 1147 dst_offs = (aligned_dst - area_dst) / page_size; 1148 step_count--; 1149 } 1150 orig_area_src = area_src; 1151 orig_area_dst = area_dst; 1152 area_src = aligned_src; 1153 area_dst = aligned_dst; 1154 } 1155 1156 /* 1157 * Read each of the pages back using the UFFD-registered mapping. We 1158 * expect that the first time we touch a page, it will result in a missing 1159 * fault. uffd_poll_thread will resolve the fault by moving source 1160 * page to destination. 1161 */ 1162 for (nr = 0; nr < step_count * step_size; nr += step_size) { 1163 unsigned long i; 1164 1165 /* Check area_src content */ 1166 for (i = 0; i < step_size; i++) { 1167 count = *area_count(area_src, nr + i); 1168 if (count != count_verify[src_offs + nr + i]) 1169 err("nr %lu source memory invalid %llu %llu\n", 1170 nr + i, count, count_verify[src_offs + nr + i]); 1171 } 1172 1173 /* Faulting into area_dst should move the page or the huge page */ 1174 for (i = 0; i < step_size; i++) { 1175 count = *area_count(area_dst, nr + i); 1176 if (count != count_verify[dst_offs + nr + i]) 1177 err("nr %lu memory corruption %llu %llu\n", 1178 nr, count, count_verify[dst_offs + nr + i]); 1179 } 1180 1181 /* Re-check area_src content which should be empty */ 1182 for (i = 0; i < step_size; i++) { 1183 count = *area_count(area_src, nr + i); 1184 if (count != 0) 1185 err("nr %lu move failed %llu %llu\n", 1186 nr, count, count_verify[src_offs + nr + i]); 1187 } 1188 } 1189 if (step_size > page_size) { 1190 area_src = orig_area_src; 1191 area_dst = orig_area_dst; 1192 } 1193 1194 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 1195 err("pipe write"); 1196 if (pthread_join(uffd_mon, NULL)) 1197 err("join() failed"); 1198 1199 if (args.missing_faults != step_count || args.minor_faults != 0) 1200 uffd_test_fail("stats check error"); 1201 else 1202 uffd_test_pass(); 1203} 1204 1205static void uffd_move_test(uffd_test_args_t *targs) 1206{ 1207 uffd_move_test_common(targs, page_size, uffd_move_handle_fault); 1208} 1209 1210static void uffd_move_pmd_test(uffd_test_args_t *targs) 1211{ 1212 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE)) 1213 err("madvise(MADV_HUGEPAGE) failure"); 1214 uffd_move_test_common(targs, read_pmd_pagesize(), 1215 uffd_move_pmd_handle_fault); 1216} 1217 1218static void uffd_move_pmd_split_test(uffd_test_args_t *targs) 1219{ 1220 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE)) 1221 err("madvise(MADV_NOHUGEPAGE) failure"); 1222 uffd_move_test_common(targs, read_pmd_pagesize(), 1223 uffd_move_pmd_handle_fault); 1224} 1225 1226static int prevent_hugepages(const char **errmsg) 1227{ 1228 /* This should be done before source area is populated */ 1229 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) { 1230 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */ 1231 if (errno != EINVAL) { 1232 if (errmsg) 1233 *errmsg = "madvise(MADV_NOHUGEPAGE) failed"; 1234 return -errno; 1235 } 1236 } 1237 return 0; 1238} 1239 1240static int request_hugepages(const char **errmsg) 1241{ 1242 /* This should be done before source area is populated */ 1243 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) { 1244 if (errmsg) { 1245 *errmsg = (errno == EINVAL) ? 1246 "CONFIG_TRANSPARENT_HUGEPAGE is not set" : 1247 "madvise(MADV_HUGEPAGE) failed"; 1248 } 1249 return -errno; 1250 } 1251 return 0; 1252} 1253 1254struct uffd_test_case_ops uffd_move_test_case_ops = { 1255 .post_alloc = prevent_hugepages, 1256}; 1257 1258struct uffd_test_case_ops uffd_move_test_pmd_case_ops = { 1259 .post_alloc = request_hugepages, 1260}; 1261 1262/* 1263 * Test the returned uffdio_register.ioctls with different register modes. 1264 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test. 1265 */ 1266static void 1267do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor) 1268{ 1269 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE); 1270 mem_type_t *mem_type = args->mem_type; 1271 int ret; 1272 1273 ret = uffd_register_with_ioctls(uffd, area_dst, page_size, 1274 miss, wp, minor, &ioctls); 1275 1276 /* 1277 * Handle special cases of UFFDIO_REGISTER here where it should 1278 * just fail with -EINVAL first.. 1279 * 1280 * Case 1: register MINOR on anon 1281 * Case 2: register with no mode selected 1282 */ 1283 if ((minor && (mem_type->mem_flag == MEM_ANON)) || 1284 (!miss && !wp && !minor)) { 1285 if (ret != -EINVAL) 1286 err("register (miss=%d, wp=%d, minor=%d) failed " 1287 "with wrong errno=%d", miss, wp, minor, ret); 1288 return; 1289 } 1290 1291 /* UFFDIO_REGISTER should succeed, then check ioctls returned */ 1292 if (miss) 1293 expected |= BIT_ULL(_UFFDIO_COPY); 1294 if (wp) 1295 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT); 1296 if (minor) 1297 expected |= BIT_ULL(_UFFDIO_CONTINUE); 1298 1299 if ((ioctls & expected) != expected) 1300 err("unexpected uffdio_register.ioctls " 1301 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", " 1302 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls); 1303 1304 if (uffd_unregister(uffd, area_dst, page_size)) 1305 err("unregister"); 1306} 1307 1308static void uffd_register_ioctls_test(uffd_test_args_t *args) 1309{ 1310 int miss, wp, minor; 1311 1312 for (miss = 0; miss <= 1; miss++) 1313 for (wp = 0; wp <= 1; wp++) 1314 for (minor = 0; minor <= 1; minor++) 1315 do_register_ioctls_test(args, miss, wp, minor); 1316 1317 uffd_test_pass(); 1318} 1319 1320uffd_test_case_t uffd_tests[] = { 1321 { 1322 /* Test returned uffdio_register.ioctls. */ 1323 .name = "register-ioctls", 1324 .uffd_fn = uffd_register_ioctls_test, 1325 .mem_targets = MEM_ALL, 1326 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS | 1327 UFFD_FEATURE_MISSING_SHMEM | 1328 UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1329 UFFD_FEATURE_WP_HUGETLBFS_SHMEM | 1330 UFFD_FEATURE_MINOR_HUGETLBFS | 1331 UFFD_FEATURE_MINOR_SHMEM, 1332 }, 1333 { 1334 .name = "zeropage", 1335 .uffd_fn = uffd_zeropage_test, 1336 .mem_targets = MEM_ALL, 1337 .uffd_feature_required = 0, 1338 }, 1339 { 1340 .name = "move", 1341 .uffd_fn = uffd_move_test, 1342 .mem_targets = MEM_ANON, 1343 .uffd_feature_required = UFFD_FEATURE_MOVE, 1344 .test_case_ops = &uffd_move_test_case_ops, 1345 }, 1346 { 1347 .name = "move-pmd", 1348 .uffd_fn = uffd_move_pmd_test, 1349 .mem_targets = MEM_ANON, 1350 .uffd_feature_required = UFFD_FEATURE_MOVE, 1351 .test_case_ops = &uffd_move_test_pmd_case_ops, 1352 }, 1353 { 1354 .name = "move-pmd-split", 1355 .uffd_fn = uffd_move_pmd_split_test, 1356 .mem_targets = MEM_ANON, 1357 .uffd_feature_required = UFFD_FEATURE_MOVE, 1358 .test_case_ops = &uffd_move_test_pmd_case_ops, 1359 }, 1360 { 1361 .name = "wp-fork", 1362 .uffd_fn = uffd_wp_fork_test, 1363 .mem_targets = MEM_ALL, 1364 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1365 UFFD_FEATURE_WP_HUGETLBFS_SHMEM, 1366 }, 1367 { 1368 .name = "wp-fork-with-event", 1369 .uffd_fn = uffd_wp_fork_with_event_test, 1370 .mem_targets = MEM_ALL, 1371 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1372 UFFD_FEATURE_WP_HUGETLBFS_SHMEM | 1373 /* when set, child process should inherit uffd-wp bits */ 1374 UFFD_FEATURE_EVENT_FORK, 1375 }, 1376 { 1377 .name = "wp-fork-pin", 1378 .uffd_fn = uffd_wp_fork_pin_test, 1379 .mem_targets = MEM_ALL, 1380 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1381 UFFD_FEATURE_WP_HUGETLBFS_SHMEM, 1382 }, 1383 { 1384 .name = "wp-fork-pin-with-event", 1385 .uffd_fn = uffd_wp_fork_pin_with_event_test, 1386 .mem_targets = MEM_ALL, 1387 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1388 UFFD_FEATURE_WP_HUGETLBFS_SHMEM | 1389 /* when set, child process should inherit uffd-wp bits */ 1390 UFFD_FEATURE_EVENT_FORK, 1391 }, 1392 { 1393 .name = "wp-unpopulated", 1394 .uffd_fn = uffd_wp_unpopulated_test, 1395 .mem_targets = MEM_ANON, 1396 .uffd_feature_required = 1397 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED, 1398 }, 1399 { 1400 .name = "minor", 1401 .uffd_fn = uffd_minor_test, 1402 .mem_targets = MEM_SHMEM | MEM_HUGETLB, 1403 .uffd_feature_required = 1404 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM, 1405 }, 1406 { 1407 .name = "minor-wp", 1408 .uffd_fn = uffd_minor_wp_test, 1409 .mem_targets = MEM_SHMEM | MEM_HUGETLB, 1410 .uffd_feature_required = 1411 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM | 1412 UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1413 /* 1414 * HACK: here we leveraged WP_UNPOPULATED to detect whether 1415 * minor mode supports wr-protect. There's no feature flag 1416 * for it so this is the best we can test against. 1417 */ 1418 UFFD_FEATURE_WP_UNPOPULATED, 1419 }, 1420 { 1421 .name = "minor-collapse", 1422 .uffd_fn = uffd_minor_collapse_test, 1423 /* MADV_COLLAPSE only works with shmem */ 1424 .mem_targets = MEM_SHMEM, 1425 /* We can't test MADV_COLLAPSE, so try our luck */ 1426 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, 1427 }, 1428 { 1429 .name = "sigbus", 1430 .uffd_fn = uffd_sigbus_test, 1431 .mem_targets = MEM_ALL, 1432 .uffd_feature_required = UFFD_FEATURE_SIGBUS | 1433 UFFD_FEATURE_EVENT_FORK, 1434 }, 1435 { 1436 .name = "sigbus-wp", 1437 .uffd_fn = uffd_sigbus_wp_test, 1438 .mem_targets = MEM_ALL, 1439 .uffd_feature_required = UFFD_FEATURE_SIGBUS | 1440 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1441 UFFD_FEATURE_WP_HUGETLBFS_SHMEM, 1442 }, 1443 { 1444 .name = "events", 1445 .uffd_fn = uffd_events_test, 1446 .mem_targets = MEM_ALL, 1447 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | 1448 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE, 1449 }, 1450 { 1451 .name = "events-wp", 1452 .uffd_fn = uffd_events_wp_test, 1453 .mem_targets = MEM_ALL, 1454 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | 1455 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE | 1456 UFFD_FEATURE_PAGEFAULT_FLAG_WP | 1457 UFFD_FEATURE_WP_HUGETLBFS_SHMEM, 1458 }, 1459 { 1460 .name = "poison", 1461 .uffd_fn = uffd_poison_test, 1462 .mem_targets = MEM_ALL, 1463 .uffd_feature_required = UFFD_FEATURE_POISON, 1464 }, 1465}; 1466 1467static void usage(const char *prog) 1468{ 1469 printf("usage: %s [-f TESTNAME]\n", prog); 1470 puts(""); 1471 puts(" -f: test name to filter (e.g., event)"); 1472 puts(" -h: show the help msg"); 1473 puts(" -l: list tests only"); 1474 puts(""); 1475 exit(KSFT_FAIL); 1476} 1477 1478int main(int argc, char *argv[]) 1479{ 1480 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t); 1481 int n_mems = sizeof(mem_types) / sizeof(mem_type_t); 1482 const char *test_filter = NULL; 1483 bool list_only = false; 1484 uffd_test_case_t *test; 1485 mem_type_t *mem_type; 1486 uffd_test_args_t args; 1487 const char *errmsg; 1488 int has_uffd, opt; 1489 int i, j; 1490 1491 while ((opt = getopt(argc, argv, "f:hl")) != -1) { 1492 switch (opt) { 1493 case 'f': 1494 test_filter = optarg; 1495 break; 1496 case 'l': 1497 list_only = true; 1498 break; 1499 case 'h': 1500 default: 1501 /* Unknown */ 1502 usage(argv[0]); 1503 break; 1504 } 1505 } 1506 1507 if (!test_filter && !list_only) { 1508 has_uffd = test_uffd_api(false); 1509 has_uffd |= test_uffd_api(true); 1510 1511 if (!has_uffd) { 1512 printf("Userfaultfd not supported or unprivileged, skip all tests\n"); 1513 exit(KSFT_SKIP); 1514 } 1515 } 1516 1517 for (i = 0; i < n_tests; i++) { 1518 test = &uffd_tests[i]; 1519 if (test_filter && !strstr(test->name, test_filter)) 1520 continue; 1521 if (list_only) { 1522 printf("%s\n", test->name); 1523 continue; 1524 } 1525 for (j = 0; j < n_mems; j++) { 1526 mem_type = &mem_types[j]; 1527 if (!(test->mem_targets & mem_type->mem_flag)) 1528 continue; 1529 1530 uffd_test_start("%s on %s", test->name, mem_type->name); 1531 if ((mem_type->mem_flag == MEM_HUGETLB || 1532 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) && 1533 (default_huge_page_size() == 0)) { 1534 uffd_test_skip("huge page size is 0, feature missing?"); 1535 continue; 1536 } 1537 if (!uffd_feature_supported(test)) { 1538 uffd_test_skip("feature missing"); 1539 continue; 1540 } 1541 if (uffd_setup_environment(&args, test, mem_type, 1542 &errmsg)) { 1543 uffd_test_skip(errmsg); 1544 continue; 1545 } 1546 test->uffd_fn(&args); 1547 uffd_test_ctx_clear(); 1548 } 1549 } 1550 1551 if (!list_only) 1552 uffd_test_report(); 1553 1554 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; 1555} 1556 1557#else /* __NR_userfaultfd */ 1558 1559#warning "missing __NR_userfaultfd definition" 1560 1561int main(void) 1562{ 1563 printf("Skipping %s (missing __NR_userfaultfd)\n", __file__); 1564 return KSFT_SKIP; 1565} 1566 1567#endif /* __NR_userfaultfd */ 1568