1#!/usr/bin/perl 2 3# test.pl 4# Run unit tests. 5 6use strict; 7use File::Basename; 8 9chdir dirname $0; 10chomp (my $DIR = `pwd`); 11 12my $TESTLIBNAME = "libobjc.A.dylib"; 13my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME"; 14 15my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build"; 16 17# xterm colors 18my $red = "\e[41;37m"; 19my $yellow = "\e[43;37m"; 20my $def = "\e[0m"; 21 22# clean, help 23if (scalar(@ARGV) == 1) { 24 my $arg = $ARGV[0]; 25 if ($arg eq "clean") { 26 my $cmd = "rm -rf $BUILDDIR *~"; 27 print "$cmd\n"; 28 `$cmd`; 29 exit 0; 30 } 31 elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") { 32 print(<<END); 33usage: $0 [options] [testname ...] 34 $0 clean 35 $0 help 36 37testname: 38 `testname` runs a specific test. If no testnames are given, runs all tests. 39 40options: 41 ARCH=<arch> 42 SDK=<sdk name> 43 ROOT=/path/to/project.roots/ 44 45 CC=<compiler name> 46 47 LANGUAGE=c,c++,objective-c,objective-c++,swift 48 MEM=mrc,arc,gc 49 STDLIB=libc++,libstdc++ 50 GUARDMALLOC=0|1 51 52 BUILD=0|1 53 RUN=0|1 54 VERBOSE=0|1 55 56examples: 57 58 test installed library, x86_64, no gc 59 $0 60 61 test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler 62 $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang 63 64 test buildit-built root with iOS simulator 65 $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator 66 67 test buildit-built root on attached iOS device 68 $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos 69END 70 exit 0; 71 } 72} 73 74######################################################################### 75## Tests 76 77my %ALL_TESTS; 78 79######################################################################### 80## Variables for use in complex build and run rules 81 82# variable # example value 83 84# things you can multiplex on the command line 85# ARCH=i386,x86_64,armv6,armv7 86# SDK=macosx,iphoneos,iphonesimulator 87# LANGUAGE=c,c++,objective-c,objective-c++,swift 88# CC=clang,gcc-4.2,llvm-gcc-4.2 89# MEM=mrc,arc,gc 90# STDLIB=libc++,libstdc++ 91# GUARDMALLOC=0,1 92 93# things you can set once on the command line 94# ROOT=/path/to/project.roots 95# BUILD=0|1 96# RUN=0|1 97# VERBOSE=0|1 98 99 100 101my $BUILD; 102my $RUN; 103my $VERBOSE; 104 105my $crashcatch = <<'END'; 106// interpose-able code to catch crashes, print, and exit cleanly 107#include <signal.h> 108#include <string.h> 109#include <unistd.h> 110 111// from dyld-interposing.h 112#define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee }; 113 114static void catchcrash(int sig) 115{ 116 const char *msg; 117 switch (sig) { 118 case SIGILL: msg = "CRASHED: SIGILL\\n"; break; 119 case SIGBUS: msg = "CRASHED: SIGBUS\\n"; break; 120 case SIGSYS: msg = "CRASHED: SIGSYS\\n"; break; 121 case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break; 122 case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break; 123 case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break; 124 default: msg = "SIG\?\?\?\?\\n"; break; 125 } 126 write(STDERR_FILENO, msg, strlen(msg)); 127 _exit(0); 128} 129 130static void setupcrash(void) __attribute__((constructor)); 131static void setupcrash(void) 132{ 133 signal(SIGILL, &catchcrash); 134 signal(SIGBUS, &catchcrash); 135 signal(SIGSYS, &catchcrash); 136 signal(SIGSEGV, &catchcrash); 137 signal(SIGTRAP, &catchcrash); 138 signal(SIGABRT, &catchcrash); 139} 140 141 142static int hacked = 0; 143ssize_t hacked_write(int fildes, const void *buf, size_t nbyte) 144{ 145 if (!hacked) { 146 setupcrash(); 147 hacked = 1; 148 } 149 return write(fildes, buf, nbyte); 150} 151 152DYLD_INTERPOSE(hacked_write, write); 153 154END 155 156 157######################################################################### 158## Harness 159 160 161# map language to buildable extensions for that language 162my %extensions_for_language = ( 163 "c" => ["c"], 164 "objective-c" => ["c", "m"], 165 "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"], 166 "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"], 167 "swift" => ["swift"], 168 169 "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"], 170 ); 171 172# map extension to languages 173my %languages_for_extension = ( 174 "c" => ["c", "objective-c", "c++", "objective-c++"], 175 "m" => ["objective-c", "objective-c++"], 176 "mm" => ["objective-c++"], 177 "cc" => ["c++", "objective-c++"], 178 "cp" => ["c++", "objective-c++"], 179 "cpp" => ["c++", "objective-c++"], 180 "cxx" => ["c++", "objective-c++"], 181 "c++" => ["c++", "objective-c++"], 182 "swift" => ["swift"], 183 ); 184 185# Run some newline-separated commands like `make` would, stopping if any fail 186# run("cmd1 \n cmd2 \n cmd3") 187sub make { 188 my $output = ""; 189 my @cmds = split("\n", $_[0]); 190 die if scalar(@cmds) == 0; 191 $? = 0; 192 foreach my $cmd (@cmds) { 193 chomp $cmd; 194 next if $cmd =~ /^\s*$/; 195 $cmd .= " 2>&1"; 196 print "$cmd\n" if $VERBOSE; 197 $output .= `$cmd`; 198 last if $?; 199 } 200 print "$output\n" if $VERBOSE; 201 return $output; 202} 203 204sub chdir_verbose { 205 my $dir = shift; 206 chdir $dir || die; 207 print "cd $dir\n" if $VERBOSE; 208} 209 210 211# Return test names from the command line. 212# Returns all tests if no tests were named. 213sub gettests { 214 my @tests; 215 216 foreach my $arg (@ARGV) { 217 push @tests, $arg if ($arg !~ /=/ && $arg !~ /^-/); 218 } 219 220 opendir(my $dir, $DIR) || die; 221 while (my $file = readdir($dir)) { 222 my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/); 223 next if ! $languages_for_extension{$ext}; 224 225 open(my $in, "< $file") || die "$file"; 226 my $contents = join "", <$in>; 227 if (defined $ALL_TESTS{$name}) { 228 print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n"; 229 } else { 230 $ALL_TESTS{$name} = $ext if ($contents =~ m#^[/*\s]*TEST_#m); 231 } 232 close($in); 233 } 234 closedir($dir); 235 236 if (scalar(@tests) == 0) { 237 @tests = keys %ALL_TESTS; 238 } 239 240 @tests = sort @tests; 241 242 return @tests; 243} 244 245 246# Turn a C compiler name into a C++ compiler name. 247sub cplusplus { 248 my ($c) = @_; 249 if ($c =~ /cc/) { 250 $c =~ s/cc/\+\+/; 251 return $c; 252 } 253 return $c . "++"; # e.g. clang => clang++ 254} 255 256# Turn a C compiler name into a Swift compiler name 257sub swift { 258 my ($c) = @_; 259 $c =~ s#[^/]*$#swift#; 260 return $c; 261} 262 263# Returns an array of all sdks from `xcodebuild -showsdks` 264my @sdks_memo; 265sub getsdks { 266 if (!@sdks_memo) { 267 @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg); 268 } 269 return @sdks_memo; 270} 271 272# Returns whether the given sdk supports -lauto 273sub supportslibauto { 274 my ($sdk) = @_; 275 return 1 if $sdk =~ /^macosx/; 276 return 0 if $sdk =~ /^iphone/; 277 die; 278} 279 280# print text with a colored prefix on each line 281sub colorprint { 282 my $color = shift; 283 while (my @lines = split("\n", shift)) { 284 for my $line (@lines) { 285 chomp $line; 286 print "$color $def$line\n"; 287 } 288 } 289} 290 291sub rewind { 292 seek($_[0], 0, 0); 293} 294 295# parse name=value,value pairs 296sub readconditions { 297 my ($conditionstring) = @_; 298 299 my %results; 300 my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g); 301 for my $condition (@conditions) { 302 my ($name, $values) = ($condition =~ /(\w+)=(.+)/); 303 $results{$name} = [split ',', $values]; 304 } 305 306 return %results; 307} 308 309# Get the name of the system SDK from sw_vers 310sub systemsdkname { 311 my @lines = `/usr/bin/sw_vers`; 312 my $name; 313 my $vers; 314 for my $line (@lines) { 315 ($name) = ($line =~ /^ProductName:\s+(.*)/) if !$name; 316 ($vers) = ($line =~ /^ProductVersion:\s+(.*)/) if !$vers; 317 } 318 319 $name =~ s/ //g; 320 $name = lc($name); 321 my $internal = ""; 322 if (-d "/usr/local/include/objc") { 323 if ($name eq "macosx") { 324 $internal = "internal"; 325 } else { 326 $internal = ".internal"; 327 } 328 } 329 return $name . $vers . $internal; 330} 331 332sub check_output { 333 my %C = %{shift()}; 334 my $name = shift; 335 my @output = @_; 336 337 my %T = %{$C{"TEST_$name"}}; 338 339 # Quietly strip MallocScribble before saving the "original" output 340 # because it is distracting. 341 filter_malloc(\@output); 342 343 my @original_output = @output; 344 345 # Run result-checking passes, reducing @output each time 346 my $xit = 1; 347 my $bad = ""; 348 my $warn = ""; 349 my $runerror = $T{TEST_RUN_OUTPUT}; 350 filter_hax(\@output); 351 filter_verbose(\@output); 352 $warn = filter_warn(\@output); 353 $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC}); 354 $bad |= filter_valgrind(\@output) if ($C{VALGRIND}); 355 $bad = filter_expected(\@output, \%C, $name) if ($bad eq ""); 356 $bad = filter_bad(\@output) if ($bad eq ""); 357 358 # OK line should be the only one left 359 $bad = "(output not 'OK: $name')" if ($bad eq "" && (scalar(@output) != 1 || $output[0] !~ /^OK: $name/)); 360 361 if ($bad ne "") { 362 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 363 colorprint($red, @original_output); 364 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 365 print "${red}FAIL: $name: $bad$def\n"; 366 $xit = 0; 367 } 368 elsif ($warn ne "") { 369 print "${yellow}PASS: /// test '$name' \\\\\\$def\n"; 370 colorprint($yellow, @original_output); 371 print "${yellow}PASS: \\\\\\ test '$name' ///$def\n"; 372 print "PASS: $name (with warnings)\n"; 373 } 374 else { 375 print "PASS: $name\n"; 376 } 377 return $xit; 378} 379 380sub filter_expected 381{ 382 my $outputref = shift; 383 my %C = %{shift()}; 384 my $name = shift; 385 386 my %T = %{$C{"TEST_$name"}}; 387 my $runerror = $T{TEST_RUN_OUTPUT} || return ""; 388 389 my $bad = ""; 390 391 my $output = join("\n", @$outputref) . "\n"; 392 if ($output !~ /$runerror/) { 393 $bad = "(run output does not match TEST_RUN_OUTPUT)"; 394 @$outputref = ("FAIL: $name"); 395 } else { 396 @$outputref = ("OK: $name"); # pacify later filter 397 } 398 399 return $bad; 400} 401 402sub filter_bad 403{ 404 my $outputref = shift; 405 my $bad = ""; 406 407 my @new_output; 408 for my $line (@$outputref) { 409 if ($line =~ /^BAD: (.*)/) { 410 $bad = "(failed)"; 411 } else { 412 push @new_output, $line; 413 } 414 } 415 416 @$outputref = @new_output; 417 return $bad; 418} 419 420sub filter_warn 421{ 422 my $outputref = shift; 423 my $warn = ""; 424 425 my @new_output; 426 for my $line (@$outputref) { 427 if ($line !~ /^WARN: (.*)/) { 428 push @new_output, $line; 429 } else { 430 $warn = "(warned)"; 431 } 432 } 433 434 @$outputref = @new_output; 435 return $warn; 436} 437 438sub filter_verbose 439{ 440 my $outputref = shift; 441 442 my @new_output; 443 for my $line (@$outputref) { 444 if ($line !~ /^VERBOSE: (.*)/) { 445 push @new_output, $line; 446 } 447 } 448 449 @$outputref = @new_output; 450} 451 452sub filter_hax 453{ 454 my $outputref = shift; 455 456 my @new_output; 457 for my $line (@$outputref) { 458 if ($line !~ /Class OS_tcp_/) { 459 push @new_output, $line; 460 } 461 } 462 463 @$outputref = @new_output; 464} 465 466sub filter_valgrind 467{ 468 my $outputref = shift; 469 my $errors = 0; 470 my $leaks = 0; 471 472 my @new_output; 473 for my $line (@$outputref) { 474 if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) { 475 # --track-origins warning (harmless) 476 next; 477 } 478 if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) { 479 # signals unsupported (harmless) 480 next; 481 } 482 if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) { 483 # signals unsupported (harmless) 484 next; 485 } 486 if ($line !~ /^^\.*==\d+==/) { 487 # not valgrind output 488 push @new_output, $line; 489 next; 490 } 491 492 my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/); 493 if (defined $errcount && $errcount > 0) { 494 $errors = 1; 495 } 496 497 (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/); 498 if (defined $leakcount && $leakcount > 0) { 499 $leaks = 1; 500 } 501 } 502 503 @$outputref = @new_output; 504 505 my $bad = ""; 506 $bad .= "(valgrind errors)" if ($errors); 507 $bad .= "(valgrind leaks)" if ($leaks); 508 return $bad; 509} 510 511 512 513sub filter_malloc 514{ 515 my $outputref = shift; 516 my $errors = 0; 517 518 my @new_output; 519 my $count = 0; 520 for my $line (@$outputref) { 521 # Ignore MallocScribble prologue. 522 # Ignore MallocStackLogging prologue. 523 if ($line =~ /malloc: enabling scribbling to detect mods to free/ || 524 $line =~ /Deleted objects will be dirtied by the collector/ || 525 $line =~ /malloc: stack logs being written into/ || 526 $line =~ /malloc: recording malloc and VM allocation stacks/) 527 { 528 next; 529 } 530 531 # not malloc output 532 push @new_output, $line; 533 534 } 535 536 @$outputref = @new_output; 537} 538 539sub filter_guardmalloc 540{ 541 my $outputref = shift; 542 my $errors = 0; 543 544 my @new_output; 545 my $count = 0; 546 for my $line (@$outputref) { 547 if ($line !~ /^GuardMalloc\[[^\]]+\]: /) { 548 # not guardmalloc output 549 push @new_output, $line; 550 next; 551 } 552 553 # Ignore 4 lines of guardmalloc prologue. 554 # Anything further is a guardmalloc error. 555 if (++$count > 4) { 556 $errors = 1; 557 } 558 } 559 560 @$outputref = @new_output; 561 562 my $bad = ""; 563 $bad .= "(guardmalloc errors)" if ($errors); 564 return $bad; 565} 566 567# TEST_SOMETHING 568# text 569# text 570# END 571sub extract_multiline { 572 my ($flag, $contents, $name) = @_; 573 if ($contents =~ /$flag\n/) { 574 my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s); 575 die "$name used $flag without END\n" if !defined($output); 576 return $output; 577 } 578 return undef; 579} 580 581 582# TEST_SOMETHING 583# text 584# OR 585# text 586# END 587sub extract_multiple_multiline { 588 my ($flag, $contents, $name) = @_; 589 if ($contents =~ /$flag\n/) { 590 my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s); 591 die "$name used $flag without END\n" if !defined($output); 592 593 $output =~ s/\nOR\n/\n|/sg; 594 $output = "^(" . $output . ")\$"; 595 return $output; 596 } 597 return undef; 598} 599 600 601sub gather_simple { 602 my $CREF = shift; 603 my %C = %{$CREF}; 604 my $name = shift; 605 chdir_verbose $DIR; 606 607 my $ext = $ALL_TESTS{$name}; 608 my $file = "$name.$ext"; 609 return 0 if !$file; 610 611 # search file for 'TEST_CONFIG' or '#include "test.h"' 612 # also collect other values: 613 # TEST_CONFIG test conditions 614 # TEST_ENV environment prefix 615 # TEST_CFLAGS compile flags 616 # TEST_BUILD build instructions 617 # TEST_BUILD_OUTPUT expected build stdout/stderr 618 # TEST_RUN_OUTPUT expected run stdout/stderr 619 open(my $in, "< $file") || die; 620 my $contents = join "", <$in>; 621 622 my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m); 623 my $disabled = ($contents =~ /\bTEST_DISABLED\b/m); 624 my $crashes = ($contents =~ /\bTEST_CRASHES\b/m); 625 my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m); 626 my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m); 627 my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m); 628 my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name); 629 my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name); 630 my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name); 631 632 return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror); 633 634 if ($disabled) { 635 print "${yellow}SKIP: $name (disabled by TEST_DISABLED)$def\n"; 636 return 0; 637 } 638 639 # check test conditions 640 641 my $run = 1; 642 my %conditions = readconditions($conditionstring); 643 if (! $conditions{LANGUAGE}) { 644 # implicit language restriction from file extension 645 $conditions{LANGUAGE} = $languages_for_extension{$ext}; 646 } 647 for my $condkey (keys %conditions) { 648 my @condvalues = @{$conditions{$condkey}}; 649 650 # special case: RUN=0 does not affect build 651 if ($condkey eq "RUN" && @condvalues == 1 && $condvalues[0] == 0) { 652 $run = 0; 653 next; 654 } 655 656 my $testvalue = $C{$condkey}; 657 next if !defined($testvalue); 658 # testvalue is the configuration being run now 659 # condvalues are the allowed values for this test 660 661 my $ok = 0; 662 for my $condvalue (@condvalues) { 663 664 # special case: objc and objc++ 665 if ($condkey eq "LANGUAGE") { 666 $condvalue = "objective-c" if $condvalue eq "objc"; 667 $condvalue = "objective-c++" if $condvalue eq "objc++"; 668 } 669 670 $ok = 1 if ($testvalue eq $condvalue); 671 672 # special case: SDK allows prefixes 673 if ($condkey eq "SDK") { 674 $ok = 1 if ($testvalue =~ /^$condvalue/); 675 } 676 677 # special case: CC and CXX allow substring matches 678 if ($condkey eq "CC" || $condkey eq "CXX") { 679 $ok = 1 if ($testvalue =~ /$condvalue/); 680 } 681 682 last if $ok; 683 } 684 685 if (!$ok) { 686 my $plural = (@condvalues > 1) ? "one of: " : ""; 687 print "SKIP: $name ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n"; 688 return 0; 689 } 690 } 691 692 # save some results for build and run phases 693 $$CREF{"TEST_$name"} = { 694 TEST_BUILD => $buildcmd, 695 TEST_BUILD_OUTPUT => $builderror, 696 TEST_CRASHES => $crashes, 697 TEST_RUN_OUTPUT => $runerror, 698 TEST_CFLAGS => $cflags, 699 TEST_ENV => $envstring, 700 TEST_RUN => $run, 701 }; 702 703 return 1; 704} 705 706# Builds a simple test 707sub build_simple { 708 my %C = %{shift()}; 709 my $name = shift; 710 my %T = %{$C{"TEST_$name"}}; 711 chdir_verbose "$C{DIR}/$name.build"; 712 713 my $ext = $ALL_TESTS{$name}; 714 my $file = "$DIR/$name.$ext"; 715 716 if ($T{TEST_CRASHES}) { 717 `echo '$crashcatch' > crashcatch.c`; 718 make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c"); 719 die "$?" if $?; 720 } 721 722 my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE} $T{TEST_CFLAGS} $file -o $name.out"; 723 724 my $output = make($cmd); 725 726 # rdar://10163155 727 $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g; 728 729 my $ok; 730 if (my $builderror = $T{TEST_BUILD_OUTPUT}) { 731 # check for expected output and ignore $? 732 if ($output =~ /$builderror/) { 733 $ok = 1; 734 } else { 735 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 736 colorprint $red, $output; 737 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 738 print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n"; 739 $ok = 0; 740 } 741 } elsif ($?) { 742 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 743 colorprint $red, $output; 744 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 745 print "${red}FAIL: $name (build failed)$def\n"; 746 $ok = 0; 747 } elsif ($output ne "") { 748 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 749 colorprint $red, $output; 750 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 751 print "${red}FAIL: $name (unexpected build output)$def\n"; 752 $ok = 0; 753 } else { 754 $ok = 1; 755 } 756 757 758 if ($ok) { 759 foreach my $file (glob("*.out *.dylib *.bundle")) { 760 make("dsymutil $file"); 761 } 762 } 763 764 return $ok; 765} 766 767# Run a simple test (testname.out, with error checking of stdout and stderr) 768sub run_simple { 769 my %C = %{shift()}; 770 my $name = shift; 771 my %T = %{$C{"TEST_$name"}}; 772 773 if (! $T{TEST_RUN}) { 774 print "PASS: $name (build only)\n"; 775 return 1; 776 } 777 else { 778 chdir_verbose "$C{DIR}/$name.build"; 779 } 780 781 my $env = "$C{ENV} $T{TEST_ENV}"; 782 if ($T{TEST_CRASHES}) { 783 $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib"; 784 } 785 786 my $output; 787 788 if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) { 789 # run on iOS device 790 791 my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build"; 792 my $remotedyld = ""; 793 $remotedyld .= " DYLD_LIBRARY_PATH=$remotedir"; 794 $remotedyld .= ":/var/root/test/" if ($C{TESTLIB} ne $TESTLIBPATH); 795 796 # elide host-specific paths 797 $env =~ s/DYLD_LIBRARY_PATH=\S+//; 798 $env =~ s/DYLD_ROOT_PATH=\S+//; 799 800 my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'"; 801 $output = make("$cmd"); 802 } 803 else { 804 # run locally 805 806 my $cmd = "env $env ./$name.out"; 807 $output = make("sh -c '$cmd 2>&1' 2>&1"); 808 # need extra sh level to capture "sh: Illegal instruction" after crash 809 # fixme fail if $? except tests that expect to crash 810 } 811 812 return check_output(\%C, $name, split("\n", $output)); 813} 814 815 816my %compiler_memo; 817sub find_compiler { 818 my ($cc, $sdk, $sdk_path) = @_; 819 820 # memoize 821 my $key = $cc . ':' . $sdk; 822 my $result = $compiler_memo{$key}; 823 return $result if defined $result; 824 825 $result = `xcrun -sdk $sdk -find $cc 2>/dev/null`; 826 827 chomp $result; 828 $compiler_memo{$key} = $result; 829 return $result; 830} 831 832sub make_one_config { 833 my $configref = shift; 834 my $root = shift; 835 my %C = %{$configref}; 836 837 # Aliases 838 $C{LANGUAGE} = "objective-c" if $C{LANGUAGE} eq "objc"; 839 $C{LANGUAGE} = "objective-c++" if $C{LANGUAGE} eq "objc++"; 840 841 # Look up SDK 842 # Try exact match first. 843 # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal"). 844 my @sdks = getsdks(); 845 if ($VERBOSE) { 846 print "Installed SDKs: @sdks\n"; 847 } 848 my $exactsdk = undef; 849 my $prefixsdk = undef; 850 foreach my $sdk (@sdks) { 851 my $SDK = $C{SDK}; 852 $exactsdk = $sdk if ($sdk eq $SDK); 853 # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2" 854 $prefixsdk = $sdk if ($sdk =~ /^$SDK[0-9]/ && $sdk gt $prefixsdk); 855 } 856 if ($exactsdk) { 857 $C{SDK} = $exactsdk; 858 } elsif ($prefixsdk) { 859 $C{SDK} = $prefixsdk; 860 } else { 861 die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n"; 862 } 863 864 # set the config name now, after massaging the language and sdk, 865 # but before adding other settings 866 my $configname = config_name(%C); 867 die if ($configname =~ /'/); 868 die if ($configname =~ / /); 869 ($C{NAME} = $configname) =~ s/~/ /g; 870 (my $configdir = $configname) =~ s#/##g; 871 $C{DIR} = "$BUILDDIR/$configdir"; 872 873 ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/); 874 875 # Look up test library (possible in root or SDK_PATH) 876 877 my $rootarg = $root; 878 my $symroot; 879 my @sympaths = ( (glob "$root/*~sym")[0], 880 (glob "$root/BuildRecords/*_install/Symbols")[0], 881 "$root/Symbols" ); 882 my @dstpaths = ( (glob "$root/*~dst")[0], 883 (glob "$root/BuildRecords/*_install/Root")[0], 884 "$root/Root" ); 885 for(my $i = 0; $i < scalar(@sympaths); $i++) { 886 if (-e $sympaths[$i] && -e $dstpaths[$i]) { 887 $symroot = $sympaths[$i]; 888 $root = $dstpaths[$i]; 889 last; 890 } 891 } 892 893 if ($root ne "" && -e "$root$C{SDK_PATH}$TESTLIBPATH") { 894 $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH"; 895 } elsif (-e "$root$TESTLIBPATH") { 896 $C{TESTLIB} = "$root$TESTLIBPATH"; 897 } elsif (-e "$root/$TESTLIBNAME") { 898 $C{TESTLIB} = "$root/$TESTLIBNAME"; 899 } else { 900 die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n" 901 # . join("\n", @dstpaths) . "\n" 902 ; 903 } 904 905 if (-e "$symroot/$TESTLIBNAME.dSYM") { 906 $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM"; 907 } 908 909 if ($VERBOSE) { 910 my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`; 911 while (my $uuid = shift @uuids) { 912 print "note: $uuid"; 913 } 914 } 915 916 # Look up compilers 917 my $cc = $C{CC}; 918 my $cxx = cplusplus($C{CC}); 919 my $swift = swift($C{CC}); 920 if (! $BUILD) { 921 $C{CC} = $cc; 922 $C{CXX} = $cxx; 923 $C{SWIFT} = $swift 924 } else { 925 $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH}); 926 $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH}); 927 $C{SWIFT} = find_compiler($swift, $C{SDK}, $C{SDK_PATH}); 928 929 die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC}; 930 die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX}; 931 die "No compiler '$swift' ('$C{SWIFT}') in SDK '$C{SDK}'\n" if !-e $C{SWIFT}; 932 } 933 934 # Populate cflags 935 936 # save-temps so dsymutil works so debug info works 937 my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} "; 938 my $objcflags = ""; 939 my $swiftflags = "-g "; 940 941 $cflags .= " -isysroot '$C{SDK_PATH}'"; 942 $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'"; 943 $swiftflags .= " -sdk '$C{SDK_PATH}'"; 944 945 my $target = ""; 946 if ($C{SDK} =~ /^iphoneos[0-9]/ && $cflags !~ /-mios-version-min/) { 947 my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9]+)/); 948 $cflags .= " -mios-version-min=$vers"; 949 $target = "$C{ARCH}-apple-ios$vers"; 950 } 951 elsif ($C{SDK} =~ /^iphonesimulator[0-9]/ && $cflags !~ /-mios-simulator-version-min/) { 952 my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9]+)/); 953 $cflags .= " -mios-simulator-version-min=$vers"; 954 $target = "$C{ARCH}-apple-ios$vers"; 955 } 956 else { 957 my ($vers) = ($C{SDK} =~ /^macosx([0-9]+\.[0-9]+)/); 958 $vers = "" if !defined($vers); 959 $target = "$C{ARCH}-apple-macosx$vers"; 960 } 961 $swiftflags .= " -target $target"; 962 963 if ($C{SDK} =~ /^iphonesimulator/ && $C{ARCH} eq "i386") { 964 $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch"; 965 } 966 967 if ($root ne "") { 968 my $library_path = dirname($C{TESTLIB}); 969 $cflags .= " -L$library_path"; 970 $cflags .= " -I '$root/usr/include'"; 971 $cflags .= " -I '$root/usr/local/include'"; 972 973 if ($C{SDK_PATH} ne "/") { 974 $cflags .= " -I '$root$C{SDK_PATH}/usr/include'"; 975 $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'"; 976 } 977 } 978 979 if ($C{CC} =~ /clang/) { 980 $cflags .= " -Qunused-arguments -fno-caret-diagnostics"; 981 $cflags .= " -stdlib=$C{STDLIB}"; # fixme -fno-objc-link-runtime" 982 $cflags .= " -Wl,-segalign,0x4000 "; 983 } 984 985 986 # Populate objcflags 987 988 $objcflags .= " -lobjc"; 989 if ($C{MEM} eq "gc") { 990 $objcflags .= " -fobjc-gc"; 991 } 992 elsif ($C{MEM} eq "arc") { 993 $objcflags .= " -fobjc-arc"; 994 } 995 elsif ($C{MEM} eq "mrc") { 996 # nothing 997 } 998 else { 999 die "unrecognized MEM '$C{MEM}'\n"; 1000 } 1001 1002 if (supportslibauto($C{SDK})) { 1003 # do this even for non-GC tests 1004 $objcflags .= " -lauto"; 1005 } 1006 1007 # Populate ENV_PREFIX 1008 $C{ENV} = "LANG=C MallocScribble=1"; 1009 $C{ENV} .= " VERBOSE=1" if $VERBOSE; 1010 if ($root ne "") { 1011 my $library_path = dirname($C{TESTLIB}); 1012 die "no spaces allowed in root" if $library_path =~ /\s+/; 1013 $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path" if ($library_path ne "/usr/lib"); 1014 } 1015 if ($C{SDK_PATH} ne "/") { 1016 die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/; 1017 $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}"; 1018 } 1019 if ($C{GUARDMALLOC}) { 1020 $ENV{GUARDMALLOC} = "1"; # checked by tests and errcheck.pl 1021 $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib"; 1022 } 1023 if ($C{SDK} =~ /^iphonesimulator[0-9]/) { 1024 my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/); 1025 $C{ENV} .= 1026 " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" . 1027 " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" . 1028 " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers"; 1029 } 1030 1031 # Populate compiler commands 1032 $C{COMPILE_C} = "env LANG=C '$C{CC}' $cflags -x c -std=gnu99"; 1033 $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++"; 1034 $C{COMPILE_M} = "env LANG=C '$C{CC}' $cflags $objcflags -x objective-c -std=gnu99"; 1035 $C{COMPILE_MM} = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++"; 1036 $C{COMPILE_SWIFT} = "env LANG=C '$C{SWIFT}' $swiftflags"; 1037 1038 $C{COMPILE} = $C{COMPILE_C} if $C{LANGUAGE} eq "c"; 1039 $C{COMPILE} = $C{COMPILE_CXX} if $C{LANGUAGE} eq "c++"; 1040 $C{COMPILE} = $C{COMPILE_M} if $C{LANGUAGE} eq "objective-c"; 1041 $C{COMPILE} = $C{COMPILE_MM} if $C{LANGUAGE} eq "objective-c++"; 1042 $C{COMPILE} = $C{COMPILE_SWIFT} if $C{LANGUAGE} eq "swift"; 1043 die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE}; 1044 1045 ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g; 1046 ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g; 1047 ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g; 1048 1049 1050 # Reject some self-inconsistent configurations 1051 if ($C{MEM} !~ /^(mrc|arc|gc)$/) { 1052 die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n"; 1053 } 1054 1055 if ($C{MEM} eq "gc" && $C{SDK} =~ /^iphone/) { 1056 print "note: skipping configuration $C{NAME}\n"; 1057 print "note: because SDK=$C{SDK} does not support MEM=$C{MEM}\n"; 1058 return 0; 1059 } 1060 if ($C{MEM} eq "arc" && $C{SDK} !~ /^iphone/ && $C{ARCH} eq "i386") { 1061 print "note: skipping configuration $C{NAME}\n"; 1062 print "note: because 32-bit Mac does not support MEM=$C{MEM}\n"; 1063 return 0; 1064 } 1065 if ($C{MEM} eq "arc" && $C{CC} !~ /clang/) { 1066 print "note: skipping configuration $C{NAME}\n"; 1067 print "note: because CC=$C{CC} does not support MEM=$C{MEM}\n"; 1068 return 0; 1069 } 1070 1071 if ($C{STDLIB} ne "libstdc++" && $C{CC} !~ /clang/) { 1072 print "note: skipping configuration $C{NAME}\n"; 1073 print "note: because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n"; 1074 return 0; 1075 } 1076 1077 # fixme 1078 if ($C{LANGUAGE} eq "swift" && $C{ARCH} =~ /^arm/) { 1079 print "note: skipping configuration $C{NAME}\n"; 1080 print "note: because ARCH=$C{ARCH} does not support LANGAUGE=SWIFT\n"; 1081 return 0; 1082 } 1083 1084 %$configref = %C; 1085} 1086 1087sub make_configs { 1088 my ($root, %args) = @_; 1089 1090 my @results = ({}); # start with one empty config 1091 1092 for my $key (keys %args) { 1093 my @newresults; 1094 my @values = @{$args{$key}}; 1095 for my $configref (@results) { 1096 my %config = %{$configref}; 1097 for my $value (@values) { 1098 my %newconfig = %config; 1099 $newconfig{$key} = $value; 1100 push @newresults, \%newconfig; 1101 } 1102 } 1103 @results = @newresults; 1104 } 1105 1106 my @newresults; 1107 for my $configref(@results) { 1108 if (make_one_config($configref, $root)) { 1109 push @newresults, $configref; 1110 } 1111 } 1112 1113 return @newresults; 1114} 1115 1116sub config_name { 1117 my %config = @_; 1118 my $name = ""; 1119 for my $key (sort keys %config) { 1120 $name .= '~' if $name ne ""; 1121 $name .= "$key=$config{$key}"; 1122 } 1123 return $name; 1124} 1125 1126sub run_one_config { 1127 my %C = %{shift()}; 1128 my @tests = @_; 1129 1130 # Build and run 1131 my $testcount = 0; 1132 my $failcount = 0; 1133 1134 my @gathertests; 1135 foreach my $test (@tests) { 1136 if ($VERBOSE) { 1137 print "\nGATHER $test\n"; 1138 } 1139 1140 if ($ALL_TESTS{$test}) { 1141 gather_simple(\%C, $test) || next; # not pass, not fail 1142 push @gathertests, $test; 1143 } else { 1144 die "No test named '$test'\n"; 1145 } 1146 } 1147 1148 my @builttests; 1149 if (!$BUILD) { 1150 @builttests = @gathertests; 1151 $testcount = scalar(@gathertests); 1152 } else { 1153 my $configdir = $C{DIR}; 1154 print $configdir, "\n" if $VERBOSE; 1155 mkdir $configdir || die; 1156 1157 foreach my $test (@gathertests) { 1158 if ($VERBOSE) { 1159 print "\nBUILD $test\n"; 1160 } 1161 mkdir "$configdir/$test.build" || die; 1162 1163 if ($ALL_TESTS{$test}) { 1164 $testcount++; 1165 if (!build_simple(\%C, $test)) { 1166 $failcount++; 1167 } else { 1168 push @builttests, $test; 1169 } 1170 } else { 1171 die "No test named '$test'\n"; 1172 } 1173 } 1174 } 1175 1176 if (!$RUN || !scalar(@builttests)) { 1177 # nothing to do 1178 } 1179 else { 1180 if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) { 1181 # upload all tests to iOS device 1182 make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/"); 1183 die "Couldn't rsync tests to device\n" if ($?); 1184 1185 # upload library to iOS device 1186 if ($C{TESTLIB} ne $TESTLIBPATH) { 1187 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTLIB} rsync://root\@localhost:10873/root/var/root/test/"); 1188 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?); 1189 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/"); 1190 } 1191 } 1192 1193 foreach my $test (@builttests) { 1194 print "\nRUN $test\n" if ($VERBOSE); 1195 1196 if ($ALL_TESTS{$test}) 1197 { 1198 if (!run_simple(\%C, $test)) { 1199 $failcount++; 1200 } 1201 } else { 1202 die "No test named '$test'\n"; 1203 } 1204 } 1205 } 1206 1207 return ($testcount, $failcount); 1208} 1209 1210 1211 1212# Return value if set by "$argname=value" on the command line 1213# Return $default if not set. 1214sub getargs { 1215 my ($argname, $default) = @_; 1216 1217 foreach my $arg (@ARGV) { 1218 my ($value) = ($arg =~ /^$argname=(.+)$/); 1219 return [split ',', $value] if defined $value; 1220 } 1221 1222 return [split ',', $default]; 1223} 1224 1225# Return 1 or 0 if set by "$argname=1" or "$argname=0" on the 1226# command line. Return $default if not set. 1227sub getbools { 1228 my ($argname, $default) = @_; 1229 1230 my @values = @{getargs($argname, $default)}; 1231 return [( map { ($_ eq "0") ? 0 : 1 } @values )]; 1232} 1233 1234sub getarg { 1235 my ($argname, $default) = @_; 1236 my @values = @{getargs($argname, $default)}; 1237 die "Only one value allowed for $argname\n" if @values > 1; 1238 return $values[0]; 1239} 1240 1241sub getbool { 1242 my ($argname, $default) = @_; 1243 my @values = @{getbools($argname, $default)}; 1244 die "Only one value allowed for $argname\n" if @values > 1; 1245 return $values[0]; 1246} 1247 1248 1249# main 1250my %args; 1251 1252 1253my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386"; 1254$args{ARCH} = getargs("ARCH", 0); 1255$args{ARCH} = getargs("ARCHS", $default_arch) if !@{$args{ARCH}}[0]; 1256 1257$args{SDK} = getargs("SDK", "macosx"); 1258 1259$args{MEM} = getargs("MEM", "mrc"); 1260$args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c,swift")} ]; 1261$args{STDLIB} = getargs("STDLIB", "libstdc++"); 1262 1263$args{CC} = getargs("CC", "clang"); 1264 1265$args{GUARDMALLOC} = getbools("GUARDMALLOC", 0); 1266 1267$BUILD = getbool("BUILD", 1); 1268$RUN = getbool("RUN", 1); 1269$VERBOSE = getbool("VERBOSE", 0); 1270 1271my $root = getarg("ROOT", ""); 1272$root =~ s#/*$##; 1273 1274my @tests = gettests(); 1275 1276print "note: -----\n"; 1277print "note: testing root '$root'\n"; 1278 1279my @configs = make_configs($root, %args); 1280 1281print "note: -----\n"; 1282print "note: testing ", scalar(@configs), " configurations:\n"; 1283for my $configref (@configs) { 1284 my $configname = $$configref{NAME}; 1285 print "note: configuration $configname\n"; 1286} 1287 1288if ($BUILD) { 1289 `rm -rf '$BUILDDIR'`; 1290 mkdir "$BUILDDIR" || die; 1291} 1292 1293my $failed = 0; 1294 1295my $testconfigs = @configs; 1296my $failconfigs = 0; 1297my $testcount = 0; 1298my $failcount = 0; 1299for my $configref (@configs) { 1300 my $configname = $$configref{NAME}; 1301 print "note: -----\n"; 1302 print "note: \nnote: $configname\nnote: \n"; 1303 1304 (my $t, my $f) = eval { run_one_config($configref, @tests); }; 1305 if ($@) { 1306 chomp $@; 1307 print "${red}FAIL: $configname${def}\n"; 1308 print "${red}FAIL: $@${def}\n"; 1309 $failconfigs++; 1310 } else { 1311 my $color = ($f ? $red : ""); 1312 print "note:\n"; 1313 print "${color}note: $configname$def\n"; 1314 print "${color}note: $t tests, $f failures$def\n"; 1315 $testcount += $t; 1316 $failcount += $f; 1317 $failconfigs++ if ($f); 1318 } 1319} 1320 1321print "note: -----\n"; 1322my $color = ($failconfigs ? $red : ""); 1323print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n"; 1324print "${color}note: $testcount tests, $failcount failures$def\n"; 1325 1326$failed = ($failconfigs ? 1 : 0); 1327 1328exit ($failed ? 1 : 0); 1329