1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2020 Google LLC 4 */ 5#define _GNU_SOURCE 6 7#include <errno.h> 8#include <stdlib.h> 9#include <stdio.h> 10#include <string.h> 11#include <sys/mman.h> 12#include <time.h> 13#include <stdbool.h> 14 15#include "../kselftest.h" 16 17#define EXPECT_SUCCESS 0 18#define EXPECT_FAILURE 1 19#define NON_OVERLAPPING 0 20#define OVERLAPPING 1 21#define NS_PER_SEC 1000000000ULL 22#define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ 23#define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ 24 25#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) 26#define SIZE_MB(m) ((size_t)m * (1024 * 1024)) 27#define SIZE_KB(k) ((size_t)k * 1024) 28 29struct config { 30 unsigned long long src_alignment; 31 unsigned long long dest_alignment; 32 unsigned long long region_size; 33 int overlapping; 34 int dest_preamble_size; 35}; 36 37struct test { 38 const char *name; 39 struct config config; 40 int expect_failure; 41}; 42 43enum { 44 _1KB = 1ULL << 10, /* 1KB -> not page aligned */ 45 _4KB = 4ULL << 10, 46 _8KB = 8ULL << 10, 47 _1MB = 1ULL << 20, 48 _2MB = 2ULL << 20, 49 _4MB = 4ULL << 20, 50 _5MB = 5ULL << 20, 51 _1GB = 1ULL << 30, 52 _2GB = 2ULL << 30, 53 PMD = _2MB, 54 PUD = _1GB, 55}; 56 57#define PTE page_size 58 59#define MAKE_TEST(source_align, destination_align, size, \ 60 overlaps, should_fail, test_name) \ 61(struct test){ \ 62 .name = test_name, \ 63 .config = { \ 64 .src_alignment = source_align, \ 65 .dest_alignment = destination_align, \ 66 .region_size = size, \ 67 .overlapping = overlaps, \ 68 }, \ 69 .expect_failure = should_fail \ 70} 71 72/* 73 * Returns false if the requested remap region overlaps with an 74 * existing mapping (e.g text, stack) else returns true. 75 */ 76static bool is_remap_region_valid(void *addr, unsigned long long size) 77{ 78 void *remap_addr = NULL; 79 bool ret = true; 80 81 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ 82 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, 83 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 84 -1, 0); 85 86 if (remap_addr == MAP_FAILED) { 87 if (errno == EEXIST) 88 ret = false; 89 } else { 90 munmap(remap_addr, size); 91 } 92 93 return ret; 94} 95 96/* Returns mmap_min_addr sysctl tunable from procfs */ 97static unsigned long long get_mmap_min_addr(void) 98{ 99 FILE *fp; 100 int n_matched; 101 static unsigned long long addr; 102 103 if (addr) 104 return addr; 105 106 fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); 107 if (fp == NULL) { 108 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", 109 strerror(errno)); 110 exit(KSFT_SKIP); 111 } 112 113 n_matched = fscanf(fp, "%llu", &addr); 114 if (n_matched != 1) { 115 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", 116 strerror(errno)); 117 fclose(fp); 118 exit(KSFT_SKIP); 119 } 120 121 fclose(fp); 122 return addr; 123} 124 125/* 126 * Using /proc/self/maps, assert that the specified address range is contained 127 * within a single mapping. 128 */ 129static bool is_range_mapped(FILE *maps_fp, void *start, void *end) 130{ 131 char *line = NULL; 132 size_t len = 0; 133 bool success = false; 134 135 rewind(maps_fp); 136 137 while (getline(&line, &len, maps_fp) != -1) { 138 char *first = strtok(line, "- "); 139 void *first_val = (void *)strtol(first, NULL, 16); 140 char *second = strtok(NULL, "- "); 141 void *second_val = (void *) strtol(second, NULL, 16); 142 143 if (first_val <= start && second_val >= end) { 144 success = true; 145 break; 146 } 147 } 148 149 return success; 150} 151 152/* 153 * Returns the start address of the mapping on success, else returns 154 * NULL on failure. 155 */ 156static void *get_source_mapping(struct config c) 157{ 158 unsigned long long addr = 0ULL; 159 void *src_addr = NULL; 160 unsigned long long mmap_min_addr; 161 162 mmap_min_addr = get_mmap_min_addr(); 163 /* 164 * For some tests, we need to not have any mappings below the 165 * source mapping. Add some headroom to mmap_min_addr for this. 166 */ 167 mmap_min_addr += 10 * _4MB; 168 169retry: 170 addr += c.src_alignment; 171 if (addr < mmap_min_addr) 172 goto retry; 173 174 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, 175 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 176 -1, 0); 177 if (src_addr == MAP_FAILED) { 178 if (errno == EPERM || errno == EEXIST) 179 goto retry; 180 goto error; 181 } 182 /* 183 * Check that the address is aligned to the specified alignment. 184 * Addresses which have alignments that are multiples of that 185 * specified are not considered valid. For instance, 1GB address is 186 * 2MB-aligned, however it will not be considered valid for a 187 * requested alignment of 2MB. This is done to reduce coincidental 188 * alignment in the tests. 189 */ 190 if (((unsigned long long) src_addr & (c.src_alignment - 1)) || 191 !((unsigned long long) src_addr & c.src_alignment)) { 192 munmap(src_addr, c.region_size); 193 goto retry; 194 } 195 196 if (!src_addr) 197 goto error; 198 199 return src_addr; 200error: 201 ksft_print_msg("Failed to map source region: %s\n", 202 strerror(errno)); 203 return NULL; 204} 205 206/* 207 * This test validates that merge is called when expanding a mapping. 208 * Mapping containing three pages is created, middle page is unmapped 209 * and then the mapping containing the first page is expanded so that 210 * it fills the created hole. The two parts should merge creating 211 * single mapping with three pages. 212 */ 213static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) 214{ 215 char *test_name = "mremap expand merge"; 216 bool success = false; 217 char *remap, *start; 218 219 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 220 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 221 222 if (start == MAP_FAILED) { 223 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 224 goto out; 225 } 226 227 munmap(start + page_size, page_size); 228 remap = mremap(start, page_size, 2 * page_size, 0); 229 if (remap == MAP_FAILED) { 230 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 231 munmap(start, page_size); 232 munmap(start + 2 * page_size, page_size); 233 goto out; 234 } 235 236 success = is_range_mapped(maps_fp, start, start + 3 * page_size); 237 munmap(start, 3 * page_size); 238 239out: 240 if (success) 241 ksft_test_result_pass("%s\n", test_name); 242 else 243 ksft_test_result_fail("%s\n", test_name); 244} 245 246/* 247 * Similar to mremap_expand_merge() except instead of removing the middle page, 248 * we remove the last then attempt to remap offset from the second page. This 249 * should result in the mapping being restored to its former state. 250 */ 251static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) 252{ 253 254 char *test_name = "mremap expand merge offset"; 255 bool success = false; 256 char *remap, *start; 257 258 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 259 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 260 261 if (start == MAP_FAILED) { 262 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 263 goto out; 264 } 265 266 /* Unmap final page to ensure we have space to expand. */ 267 munmap(start + 2 * page_size, page_size); 268 remap = mremap(start + page_size, page_size, 2 * page_size, 0); 269 if (remap == MAP_FAILED) { 270 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 271 munmap(start, 2 * page_size); 272 goto out; 273 } 274 275 success = is_range_mapped(maps_fp, start, start + 3 * page_size); 276 munmap(start, 3 * page_size); 277 278out: 279 if (success) 280 ksft_test_result_pass("%s\n", test_name); 281 else 282 ksft_test_result_fail("%s\n", test_name); 283} 284 285/* 286 * Verify that an mremap within a range does not cause corruption 287 * of unrelated part of range. 288 * 289 * Consider the following range which is 2MB aligned and is 290 * a part of a larger 20MB range which is not shown. Each 291 * character is 256KB below making the source and destination 292 * 2MB each. The lower case letters are moved (s to d) and the 293 * upper case letters are not moved. The below test verifies 294 * that the upper case S letters are not corrupted by the 295 * adjacent mremap. 296 * 297 * |DDDDddddSSSSssss| 298 */ 299static void mremap_move_within_range(char pattern_seed) 300{ 301 char *test_name = "mremap mremap move within range"; 302 void *src, *dest; 303 int i, success = 1; 304 305 size_t size = SIZE_MB(20); 306 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 307 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 308 if (ptr == MAP_FAILED) { 309 perror("mmap"); 310 success = 0; 311 goto out; 312 } 313 memset(ptr, 0, size); 314 315 src = ptr + SIZE_MB(6); 316 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); 317 318 /* Set byte pattern for source block. */ 319 srand(pattern_seed); 320 for (i = 0; i < SIZE_MB(2); i++) { 321 ((char *)src)[i] = (char) rand(); 322 } 323 324 dest = src - SIZE_MB(2); 325 326 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 327 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 328 if (new_ptr == MAP_FAILED) { 329 perror("mremap"); 330 success = 0; 331 goto out; 332 } 333 334 /* Verify byte pattern after remapping */ 335 srand(pattern_seed); 336 for (i = 0; i < SIZE_MB(1); i++) { 337 char c = (char) rand(); 338 339 if (((char *)src)[i] != c) { 340 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 341 i); 342 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 343 ((char *) src)[i] & 0xff); 344 success = 0; 345 } 346 } 347 348out: 349 if (munmap(ptr, size) == -1) 350 perror("munmap"); 351 352 if (success) 353 ksft_test_result_pass("%s\n", test_name); 354 else 355 ksft_test_result_fail("%s\n", test_name); 356} 357 358/* Returns the time taken for the remap on success else returns -1. */ 359static long long remap_region(struct config c, unsigned int threshold_mb, 360 char pattern_seed) 361{ 362 void *addr, *src_addr, *dest_addr, *dest_preamble_addr; 363 int d; 364 unsigned long long t; 365 struct timespec t_start = {0, 0}, t_end = {0, 0}; 366 long long start_ns, end_ns, align_mask, ret, offset; 367 unsigned long long threshold; 368 369 if (threshold_mb == VALIDATION_NO_THRESHOLD) 370 threshold = c.region_size; 371 else 372 threshold = MIN(threshold_mb * _1MB, c.region_size); 373 374 src_addr = get_source_mapping(c); 375 if (!src_addr) { 376 ret = -1; 377 goto out; 378 } 379 380 /* Set byte pattern for source block. */ 381 srand(pattern_seed); 382 for (t = 0; t < threshold; t++) 383 memset((char *) src_addr + t, (char) rand(), 1); 384 385 /* Mask to zero out lower bits of address for alignment */ 386 align_mask = ~(c.dest_alignment - 1); 387 /* Offset of destination address from the end of the source region */ 388 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; 389 addr = (void *) (((unsigned long long) src_addr + c.region_size 390 + offset) & align_mask); 391 392 /* Remap after the destination block preamble. */ 393 addr += c.dest_preamble_size; 394 395 /* See comment in get_source_mapping() */ 396 if (!((unsigned long long) addr & c.dest_alignment)) 397 addr = (void *) ((unsigned long long) addr | c.dest_alignment); 398 399 /* Don't destroy existing mappings unless expected to overlap */ 400 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { 401 /* Check for unsigned overflow */ 402 if (addr + c.dest_alignment < addr) { 403 ksft_print_msg("Couldn't find a valid region to remap to\n"); 404 ret = -1; 405 goto clean_up_src; 406 } 407 addr += c.dest_alignment; 408 } 409 410 if (c.dest_preamble_size) { 411 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, 412 PROT_READ | PROT_WRITE, 413 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 414 -1, 0); 415 if (dest_preamble_addr == MAP_FAILED) { 416 ksft_print_msg("Failed to map dest preamble region: %s\n", 417 strerror(errno)); 418 ret = -1; 419 goto clean_up_src; 420 } 421 422 /* Set byte pattern for the dest preamble block. */ 423 srand(pattern_seed); 424 for (d = 0; d < c.dest_preamble_size; d++) 425 memset((char *) dest_preamble_addr + d, (char) rand(), 1); 426 } 427 428 clock_gettime(CLOCK_MONOTONIC, &t_start); 429 dest_addr = mremap(src_addr, c.region_size, c.region_size, 430 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); 431 clock_gettime(CLOCK_MONOTONIC, &t_end); 432 433 if (dest_addr == MAP_FAILED) { 434 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 435 ret = -1; 436 goto clean_up_dest_preamble; 437 } 438 439 /* Verify byte pattern after remapping */ 440 srand(pattern_seed); 441 for (t = 0; t < threshold; t++) { 442 char c = (char) rand(); 443 444 if (((char *) dest_addr)[t] != c) { 445 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 446 t); 447 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 448 ((char *) dest_addr)[t] & 0xff); 449 ret = -1; 450 goto clean_up_dest; 451 } 452 } 453 454 /* Verify the dest preamble byte pattern after remapping */ 455 if (c.dest_preamble_size) { 456 srand(pattern_seed); 457 for (d = 0; d < c.dest_preamble_size; d++) { 458 char c = (char) rand(); 459 460 if (((char *) dest_preamble_addr)[d] != c) { 461 ksft_print_msg("Preamble data after remap doesn't match at offset %d\n", 462 d); 463 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 464 ((char *) dest_preamble_addr)[d] & 0xff); 465 ret = -1; 466 goto clean_up_dest; 467 } 468 } 469 } 470 471 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; 472 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; 473 ret = end_ns - start_ns; 474 475/* 476 * Since the destination address is specified using MREMAP_FIXED, subsequent 477 * mremap will unmap any previous mapping at the address range specified by 478 * dest_addr and region_size. This significantly affects the remap time of 479 * subsequent tests. So we clean up mappings after each test. 480 */ 481clean_up_dest: 482 munmap(dest_addr, c.region_size); 483clean_up_dest_preamble: 484 if (c.dest_preamble_size && dest_preamble_addr) 485 munmap(dest_preamble_addr, c.dest_preamble_size); 486clean_up_src: 487 munmap(src_addr, c.region_size); 488out: 489 return ret; 490} 491 492/* 493 * Verify that an mremap aligning down does not destroy 494 * the beginning of the mapping just because the aligned 495 * down address landed on a mapping that maybe does not exist. 496 */ 497static void mremap_move_1mb_from_start(char pattern_seed) 498{ 499 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; 500 void *src = NULL, *dest = NULL; 501 int i, success = 1; 502 503 /* Config to reuse get_source_mapping() to do an aligned mmap. */ 504 struct config c = { 505 .src_alignment = SIZE_MB(1) + SIZE_KB(256), 506 .region_size = SIZE_MB(6) 507 }; 508 509 src = get_source_mapping(c); 510 if (!src) { 511 success = 0; 512 goto out; 513 } 514 515 c.src_alignment = SIZE_MB(1) + SIZE_KB(256); 516 dest = get_source_mapping(c); 517 if (!dest) { 518 success = 0; 519 goto out; 520 } 521 522 /* Set byte pattern for source block. */ 523 srand(pattern_seed); 524 for (i = 0; i < SIZE_MB(2); i++) { 525 ((char *)src)[i] = (char) rand(); 526 } 527 528 /* 529 * Unmap the beginning of dest so that the aligned address 530 * falls on no mapping. 531 */ 532 munmap(dest, SIZE_MB(1)); 533 534 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 535 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 536 if (new_ptr == MAP_FAILED) { 537 perror("mremap"); 538 success = 0; 539 goto out; 540 } 541 542 /* Verify byte pattern after remapping */ 543 srand(pattern_seed); 544 for (i = 0; i < SIZE_MB(1); i++) { 545 char c = (char) rand(); 546 547 if (((char *)src)[i] != c) { 548 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 549 i); 550 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 551 ((char *) src)[i] & 0xff); 552 success = 0; 553 } 554 } 555 556out: 557 if (src && munmap(src, c.region_size) == -1) 558 perror("munmap src"); 559 560 if (dest && munmap(dest, c.region_size) == -1) 561 perror("munmap dest"); 562 563 if (success) 564 ksft_test_result_pass("%s\n", test_name); 565 else 566 ksft_test_result_fail("%s\n", test_name); 567} 568 569static void run_mremap_test_case(struct test test_case, int *failures, 570 unsigned int threshold_mb, 571 unsigned int pattern_seed) 572{ 573 long long remap_time = remap_region(test_case.config, threshold_mb, 574 pattern_seed); 575 576 if (remap_time < 0) { 577 if (test_case.expect_failure) 578 ksft_test_result_xfail("%s\n\tExpected mremap failure\n", 579 test_case.name); 580 else { 581 ksft_test_result_fail("%s\n", test_case.name); 582 *failures += 1; 583 } 584 } else { 585 /* 586 * Comparing mremap time is only applicable if entire region 587 * was faulted in. 588 */ 589 if (threshold_mb == VALIDATION_NO_THRESHOLD || 590 test_case.config.region_size <= threshold_mb * _1MB) 591 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", 592 test_case.name, remap_time); 593 else 594 ksft_test_result_pass("%s\n", test_case.name); 595 } 596} 597 598static void usage(const char *cmd) 599{ 600 fprintf(stderr, 601 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" 602 "-t\t only validate threshold_mb of the remapped region\n" 603 " \t if 0 is supplied no threshold is used; all tests\n" 604 " \t are run and remapped regions validated fully.\n" 605 " \t The default threshold used is 4MB.\n" 606 "-p\t provide a seed to generate the random pattern for\n" 607 " \t validating the remapped region.\n", cmd); 608} 609 610static int parse_args(int argc, char **argv, unsigned int *threshold_mb, 611 unsigned int *pattern_seed) 612{ 613 const char *optstr = "t:p:"; 614 int opt; 615 616 while ((opt = getopt(argc, argv, optstr)) != -1) { 617 switch (opt) { 618 case 't': 619 *threshold_mb = atoi(optarg); 620 break; 621 case 'p': 622 *pattern_seed = atoi(optarg); 623 break; 624 default: 625 usage(argv[0]); 626 return -1; 627 } 628 } 629 630 if (optind < argc) { 631 usage(argv[0]); 632 return -1; 633 } 634 635 return 0; 636} 637 638#define MAX_TEST 15 639#define MAX_PERF_TEST 3 640int main(int argc, char **argv) 641{ 642 int failures = 0; 643 int i, run_perf_tests; 644 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; 645 unsigned int pattern_seed; 646 int num_expand_tests = 2; 647 int num_misc_tests = 2; 648 struct test test_cases[MAX_TEST] = {}; 649 struct test perf_test_cases[MAX_PERF_TEST]; 650 int page_size; 651 time_t t; 652 FILE *maps_fp; 653 654 pattern_seed = (unsigned int) time(&t); 655 656 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) 657 exit(EXIT_FAILURE); 658 659 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", 660 threshold_mb, pattern_seed); 661 662 page_size = sysconf(_SC_PAGESIZE); 663 664 /* Expected mremap failures */ 665 test_cases[0] = MAKE_TEST(page_size, page_size, page_size, 666 OVERLAPPING, EXPECT_FAILURE, 667 "mremap - Source and Destination Regions Overlapping"); 668 669 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, 670 NON_OVERLAPPING, EXPECT_FAILURE, 671 "mremap - Destination Address Misaligned (1KB-aligned)"); 672 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, 673 NON_OVERLAPPING, EXPECT_FAILURE, 674 "mremap - Source Address Misaligned (1KB-aligned)"); 675 676 /* Src addr PTE aligned */ 677 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, 678 NON_OVERLAPPING, EXPECT_SUCCESS, 679 "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); 680 681 /* Src addr 1MB aligned */ 682 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 683 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); 684 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 685 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 686 687 /* Src addr PMD aligned */ 688 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 689 "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); 690 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 691 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); 692 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 693 "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); 694 695 /* Src addr PUD aligned */ 696 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 697 "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); 698 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 699 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); 700 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 701 "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); 702 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 703 "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); 704 705 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 706 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 707 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 708 709 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 710 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 711 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); 712 test_cases[14].config.dest_preamble_size = 10 * _4MB; 713 714 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 715 "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); 716 /* 717 * mremap 1GB region - Page table level aligned time 718 * comparison. 719 */ 720 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 721 "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); 722 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 723 "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); 724 725 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || 726 (threshold_mb * _1MB >= _1GB); 727 728 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? 729 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); 730 731 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 732 run_mremap_test_case(test_cases[i], &failures, threshold_mb, 733 pattern_seed); 734 735 maps_fp = fopen("/proc/self/maps", "r"); 736 737 if (maps_fp == NULL) { 738 ksft_print_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); 739 exit(KSFT_FAIL); 740 } 741 742 mremap_expand_merge(maps_fp, page_size); 743 mremap_expand_merge_offset(maps_fp, page_size); 744 745 fclose(maps_fp); 746 747 mremap_move_within_range(pattern_seed); 748 mremap_move_1mb_from_start(pattern_seed); 749 750 if (run_perf_tests) { 751 ksft_print_msg("\n%s\n", 752 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); 753 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) 754 run_mremap_test_case(perf_test_cases[i], &failures, 755 threshold_mb, pattern_seed); 756 } 757 758 if (failures > 0) 759 ksft_exit_fail(); 760 else 761 ksft_exit_pass(); 762} 763