1#!/usr/bin/perl 2 3# test.pl 4# Run unit tests. 5 6use strict; 7use File::Basename; 8 9chdir dirname $0; 10chomp (my $DIR = `pwd`); 11 12my $TESTLIBNAME = "libobjc.A.dylib"; 13my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME"; 14 15my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build"; 16 17# xterm colors 18my $red = "\e[41;37m"; 19my $yellow = "\e[43;37m"; 20my $def = "\e[0m"; 21 22# clean, help 23if (scalar(@ARGV) == 1) { 24 my $arg = $ARGV[0]; 25 if ($arg eq "clean") { 26 my $cmd = "rm -rf $BUILDDIR *~"; 27 print "$cmd\n"; 28 `$cmd`; 29 exit 0; 30 } 31 elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") { 32 print(<<END); 33usage: $0 [options] [testname ...] 34 $0 clean 35 $0 help 36 37testname: 38 `testname` runs a specific test. If no testnames are given, runs all tests. 39 40options: 41 ARCH=<arch> 42 SDK=<sdk name> 43 ROOT=/path/to/project.roots/ 44 45 CC=<compiler name> 46 47 MEM=mrc,arc,gc 48 STDLIB=libc++,libstdc++ 49 GUARDMALLOC=0|1 50 51 BUILD=0|1 52 RUN=0|1 53 VERBOSE=0|1 54 55examples: 56 57 test installed library, x86_64, no gc 58 $0 59 60 test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler 61 $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang 62 63 test buildit-built root with iOS simulator 64 $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator 65 66 test buildit-built root on attached iOS device 67 $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos 68END 69 exit 0; 70 } 71} 72 73######################################################################### 74## Tests 75 76my %ALL_TESTS; 77 78######################################################################### 79## Variables for use in complex build and run rules 80 81# variable # example value 82 83# things you can multiplex on the command line 84# ARCH=i386,x86_64,armv6,armv7 85# SDK=system,macosx,iphoneos,iphonesimulator 86# LANGUAGE=c,c++,objective-c,objective-c++ 87# CC=clang,gcc-4.2,llvm-gcc-4.2 88# MEM=mrc,arc,gc 89# STDLIB=libc++,libstdc++ 90# GUARDMALLOC=0,1 91 92# things you can set once on the command line 93# ROOT=/path/to/project.roots 94# BUILD=0|1 95# RUN=0|1 96# VERBOSE=0|1 97 98 99 100my $BUILD; 101my $RUN; 102my $VERBOSE; 103 104my $crashcatch = <<'END'; 105// interpose-able code to catch crashes, print, and exit cleanly 106#include <signal.h> 107#include <string.h> 108#include <unistd.h> 109 110// from dyld-interposing.h 111#define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee }; 112 113static void catchcrash(int sig) 114{ 115 const char *msg; 116 switch (sig) { 117 case SIGILL: msg = "CRASHED: SIGILL\\n"; break; 118 case SIGBUS: msg = "CRASHED: SIGBUS\\n"; break; 119 case SIGSYS: msg = "CRASHED: SIGSYS\\n"; break; 120 case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break; 121 case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break; 122 case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break; 123 default: msg = "SIG\?\?\?\?\\n"; break; 124 } 125 write(STDERR_FILENO, msg, strlen(msg)); 126 _exit(0); 127} 128 129static void setupcrash(void) __attribute__((constructor)); 130static void setupcrash(void) 131{ 132 signal(SIGILL, &catchcrash); 133 signal(SIGBUS, &catchcrash); 134 signal(SIGSYS, &catchcrash); 135 signal(SIGSEGV, &catchcrash); 136 signal(SIGTRAP, &catchcrash); 137 signal(SIGABRT, &catchcrash); 138} 139 140 141static int hacked = 0; 142ssize_t hacked_write(int fildes, const void *buf, size_t nbyte) 143{ 144 if (!hacked) { 145 setupcrash(); 146 hacked = 1; 147 } 148 return write(fildes, buf, nbyte); 149} 150 151DYLD_INTERPOSE(hacked_write, write); 152 153END 154 155 156######################################################################### 157## Harness 158 159 160# map language to buildable extensions for that language 161my %extensions_for_language = ( 162 "c" => ["c"], 163 "objective-c" => ["c", "m"], 164 "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"], 165 "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"], 166 167 "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"], 168 ); 169 170# map extension to languages 171my %languages_for_extension = ( 172 "c" => ["c", "objective-c", "c++", "objective-c++"], 173 "m" => ["objective-c", "objective-c++"], 174 "mm" => ["objective-c++"], 175 "cc" => ["c++", "objective-c++"], 176 "cp" => ["c++", "objective-c++"], 177 "cpp" => ["c++", "objective-c++"], 178 "cxx" => ["c++", "objective-c++"], 179 "c++" => ["c++", "objective-c++"], 180 ); 181 182# Run some newline-separated commands like `make` would, stopping if any fail 183# run("cmd1 \n cmd2 \n cmd3") 184sub make { 185 my $output = ""; 186 my @cmds = split("\n", $_[0]); 187 die if scalar(@cmds) == 0; 188 $? = 0; 189 foreach my $cmd (@cmds) { 190 chomp $cmd; 191 next if $cmd =~ /^\s*$/; 192 $cmd .= " 2>&1"; 193 print "$cmd\n" if $VERBOSE; 194 $output .= `$cmd`; 195 last if $?; 196 } 197 print "$output\n" if $VERBOSE; 198 return $output; 199} 200 201sub chdir_verbose { 202 my $dir = shift; 203 chdir $dir || die; 204 print "cd $dir\n" if $VERBOSE; 205} 206 207 208# Return test names from the command line. 209# Returns all tests if no tests were named. 210sub gettests { 211 my @tests; 212 213 foreach my $arg (@ARGV) { 214 push @tests, $arg if ($arg !~ /=/ && $arg !~ /^-/); 215 } 216 217 opendir(my $dir, $DIR) || die; 218 while (my $file = readdir($dir)) { 219 my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/); 220 next if ! $languages_for_extension{$ext}; 221 222 open(my $in, "< $file") || die "$file"; 223 my $contents = join "", <$in>; 224 if (defined $ALL_TESTS{$name}) { 225 print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n"; 226 } else { 227 $ALL_TESTS{$name} = $ext if ($contents =~ m#^[/*\s]*TEST_#m); 228 } 229 close($in); 230 } 231 closedir($dir); 232 233 if (scalar(@tests) == 0) { 234 @tests = keys %ALL_TESTS; 235 } 236 237 @tests = sort @tests; 238 239 return @tests; 240} 241 242 243# Turn a C compiler name into a C++ compiler name. 244sub cplusplus { 245 my ($c) = @_; 246 if ($c =~ /cc/) { 247 $c =~ s/cc/\+\+/; 248 return $c; 249 } 250 return $c . "++"; # e.g. clang => clang++ 251} 252 253# Returns an array of all sdks from `xcodebuild -showsdks` 254my @sdks_memo; 255sub getsdks { 256 if (!@sdks_memo) { 257 @sdks_memo = ("system", `xcodebuild -showsdks` =~ /-sdk (.+)$/mg); 258 } 259 return @sdks_memo; 260} 261 262# Returns whether the given sdk supports -lauto 263sub supportslibauto { 264 my ($sdk) = @_; 265 return 1 if $sdk eq "system"; 266 return 1 if $sdk =~ /^macosx/; 267 return 0 if $sdk =~ /^iphone/; 268 die; 269} 270 271# print text with a colored prefix on each line 272sub colorprint { 273 my $color = shift; 274 while (my @lines = split("\n", shift)) { 275 for my $line (@lines) { 276 chomp $line; 277 print "$color $def$line\n"; 278 } 279 } 280} 281 282sub rewind { 283 seek($_[0], 0, 0); 284} 285 286# parse name=value,value pairs 287sub readconditions { 288 my ($conditionstring) = @_; 289 290 my %results; 291 my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g); 292 for my $condition (@conditions) { 293 my ($name, $values) = ($condition =~ /(\w+)=(.+)/); 294 $results{$name} = [split ',', $values]; 295 } 296 297 return %results; 298} 299 300# Get the name of the system SDK from sw_vers 301sub systemsdkname { 302 my @lines = `/usr/bin/sw_vers`; 303 my $name; 304 my $vers; 305 for my $line (@lines) { 306 ($name) = ($line =~ /^ProductName:\s+(.*)/) if !$name; 307 ($vers) = ($line =~ /^ProductVersion:\s+(.*)/) if !$vers; 308 } 309 310 $name =~ s/ //g; 311 $name = lc($name); 312 my $internal = ""; 313 if (-d "/usr/local/include/objc") { 314 if ($name eq "macosx") { 315 $internal = "internal"; 316 } else { 317 $internal = ".internal"; 318 } 319 } 320 return $name . $vers . $internal; 321} 322 323sub check_output { 324 my %C = %{shift()}; 325 my $name = shift; 326 my @output = @_; 327 328 my %T = %{$C{"TEST_$name"}}; 329 330 # Quietly strip MallocScribble before saving the "original" output 331 # because it is distracting. 332 filter_malloc(\@output); 333 334 my @original_output = @output; 335 336 # Run result-checking passes, reducing @output each time 337 my $xit = 1; 338 my $bad = ""; 339 my $warn = ""; 340 my $runerror = $T{TEST_RUN_OUTPUT}; 341 filter_verbose(\@output); 342 $warn = filter_warn(\@output); 343 $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC}); 344 $bad |= filter_valgrind(\@output) if ($C{VALGRIND}); 345 $bad = filter_expected(\@output, \%C, $name) if ($bad eq ""); 346 $bad = filter_bad(\@output) if ($bad eq ""); 347 348 # OK line should be the only one left 349 $bad = "(output not 'OK: $name')" if ($bad eq "" && (scalar(@output) != 1 || $output[0] !~ /^OK: $name/)); 350 351 if ($bad ne "") { 352 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 353 colorprint($red, @original_output); 354 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 355 print "${red}FAIL: $name: $bad$def\n"; 356 $xit = 0; 357 } 358 elsif ($warn ne "") { 359 print "${yellow}PASS: /// test '$name' \\\\\\$def\n"; 360 colorprint($yellow, @original_output); 361 print "${yellow}PASS: \\\\\\ test '$name' ///$def\n"; 362 print "PASS: $name (with warnings)\n"; 363 } 364 else { 365 print "PASS: $name\n"; 366 } 367 return $xit; 368} 369 370sub filter_expected 371{ 372 my $outputref = shift; 373 my %C = %{shift()}; 374 my $name = shift; 375 376 my %T = %{$C{"TEST_$name"}}; 377 my $runerror = $T{TEST_RUN_OUTPUT} || return ""; 378 379 my $bad = ""; 380 381 my $output = join("\n", @$outputref) . "\n"; 382 if ($output !~ /$runerror/) { 383 $bad = "(run output does not match TEST_RUN_OUTPUT)"; 384 @$outputref = ("FAIL: $name"); 385 } else { 386 @$outputref = ("OK: $name"); # pacify later filter 387 } 388 389 return $bad; 390} 391 392sub filter_bad 393{ 394 my $outputref = shift; 395 my $bad = ""; 396 397 my @new_output; 398 for my $line (@$outputref) { 399 if ($line =~ /^BAD: (.*)/) { 400 $bad = "(failed)"; 401 } else { 402 push @new_output, $line; 403 } 404 } 405 406 @$outputref = @new_output; 407 return $bad; 408} 409 410sub filter_warn 411{ 412 my $outputref = shift; 413 my $warn = ""; 414 415 my @new_output; 416 for my $line (@$outputref) { 417 if ($line !~ /^WARN: (.*)/) { 418 push @new_output, $line; 419 } else { 420 $warn = "(warned)"; 421 } 422 } 423 424 @$outputref = @new_output; 425 return $warn; 426} 427 428sub filter_verbose 429{ 430 my $outputref = shift; 431 432 my @new_output; 433 for my $line (@$outputref) { 434 if ($line !~ /^VERBOSE: (.*)/) { 435 push @new_output, $line; 436 } 437 } 438 439 @$outputref = @new_output; 440} 441 442sub filter_valgrind 443{ 444 my $outputref = shift; 445 my $errors = 0; 446 my $leaks = 0; 447 448 my @new_output; 449 for my $line (@$outputref) { 450 if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) { 451 # --track-origins warning (harmless) 452 next; 453 } 454 if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) { 455 # signals unsupported (harmless) 456 next; 457 } 458 if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) { 459 # signals unsupported (harmless) 460 next; 461 } 462 if ($line !~ /^^\.*==\d+==/) { 463 # not valgrind output 464 push @new_output, $line; 465 next; 466 } 467 468 my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/); 469 if (defined $errcount && $errcount > 0) { 470 $errors = 1; 471 } 472 473 (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/); 474 if (defined $leakcount && $leakcount > 0) { 475 $leaks = 1; 476 } 477 } 478 479 @$outputref = @new_output; 480 481 my $bad = ""; 482 $bad .= "(valgrind errors)" if ($errors); 483 $bad .= "(valgrind leaks)" if ($leaks); 484 return $bad; 485} 486 487 488 489sub filter_malloc 490{ 491 my $outputref = shift; 492 my $errors = 0; 493 494 my @new_output; 495 my $count = 0; 496 for my $line (@$outputref) { 497 # Ignore MallocScribble prologue. 498 # Ignore MallocStackLogging prologue. 499 if ($line =~ /malloc: enabling scribbling to detect mods to free/ || 500 $line =~ /Deleted objects will be dirtied by the collector/ || 501 $line =~ /malloc: stack logs being written into/ || 502 $line =~ /malloc: recording malloc and VM allocation stacks/) 503 { 504 next; 505 } 506 507 # not malloc output 508 push @new_output, $line; 509 510 } 511 512 @$outputref = @new_output; 513} 514 515sub filter_guardmalloc 516{ 517 my $outputref = shift; 518 my $errors = 0; 519 520 my @new_output; 521 my $count = 0; 522 for my $line (@$outputref) { 523 if ($line !~ /^GuardMalloc\[[^\]]+\]: /) { 524 # not guardmalloc output 525 push @new_output, $line; 526 next; 527 } 528 529 # Ignore 4 lines of guardmalloc prologue. 530 # Anything further is a guardmalloc error. 531 if (++$count > 4) { 532 $errors = 1; 533 } 534 } 535 536 @$outputref = @new_output; 537 538 my $bad = ""; 539 $bad .= "(guardmalloc errors)" if ($errors); 540 return $bad; 541} 542 543sub gather_simple { 544 my $CREF = shift; 545 my %C = %{$CREF}; 546 my $name = shift; 547 chdir_verbose $DIR; 548 549 my $ext = $ALL_TESTS{$name}; 550 my $file = "$name.$ext"; 551 return 0 if !$file; 552 553 # search file for 'TEST_CONFIG' or '#include "test.h"' 554 # also collect other values: 555 # TEST_CONFIG test conditions 556 # TEST_ENV environment prefix 557 # TEST_CFLAGS compile flags 558 # TEST_BUILD build instructions 559 # TEST_BUILD_OUTPUT expected build stdout/stderr 560 # TEST_RUN_OUTPUT expected run stdout/stderr 561 open(my $in, "< $file") || die; 562 my $contents = join "", <$in>; 563 564 my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m); 565 my $disabled = ($contents =~ /\bTEST_DISABLED\b/m); 566 my $crashes = ($contents =~ /\bTEST_CRASHES\b/m); 567 my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m); 568 my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m); 569 my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m); 570 my ($buildcmd) = ($contents =~ /TEST_BUILD\n(.*?\n)END[ *\/]*\n/s); 571 my ($builderror) = ($contents =~ /TEST_BUILD_OUTPUT\n(.*?\n)END[ *\/]*\n/s); 572 my ($runerror) = ($contents =~ /TEST_RUN_OUTPUT\n(.*?\n)END[ *\/]*\n/s); 573 574 return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror); 575 576 if ($disabled) { 577 print "${yellow}SKIP: $name (disabled by TEST_DISABLED)$def\n"; 578 return 0; 579 } 580 581 # check test conditions 582 583 my $run = 1; 584 my %conditions = readconditions($conditionstring); 585 if (! $conditions{LANGUAGE}) { 586 # implicit language restriction from file extension 587 $conditions{LANGUAGE} = $languages_for_extension{$ext}; 588 } 589 for my $condkey (keys %conditions) { 590 my @condvalues = @{$conditions{$condkey}}; 591 592 # special case: RUN=0 does not affect build 593 if ($condkey eq "RUN" && @condvalues == 1 && $condvalues[0] == 0) { 594 $run = 0; 595 next; 596 } 597 598 my $testvalue = $C{$condkey}; 599 next if !defined($testvalue); 600 # testvalue is the configuration being run now 601 # condvalues are the allowed values for this test 602 603 # special case: look up the name of SDK "system" 604 if ($condkey eq "SDK" && $testvalue eq "system") { 605 $testvalue = systemsdkname(); 606 } 607 608 my $ok = 0; 609 for my $condvalue (@condvalues) { 610 611 # special case: objc and objc++ 612 if ($condkey eq "LANGUAGE") { 613 $condvalue = "objective-c" if $condvalue eq "objc"; 614 $condvalue = "objective-c++" if $condvalue eq "objc++"; 615 } 616 617 $ok = 1 if ($testvalue eq $condvalue); 618 619 # special case: SDK allows prefixes, and "system" is "macosx" 620 if ($condkey eq "SDK") { 621 $ok = 1 if ($testvalue =~ /^$condvalue/); 622 $ok = 1 if ($testvalue eq "system" && "macosx" =~ /^$condvalue/); 623 } 624 625 # special case: CC and CXX allow substring matches 626 if ($condkey eq "CC" || $condkey eq "CXX") { 627 $ok = 1 if ($testvalue =~ /$condvalue/); 628 } 629 630 last if $ok; 631 } 632 633 if (!$ok) { 634 my $plural = (@condvalues > 1) ? "one of: " : ""; 635 print "SKIP: $name ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n"; 636 return 0; 637 } 638 } 639 640 # builderror is multiple REs separated by OR 641 if (defined $builderror) { 642 $builderror =~ s/\nOR\n/\n|/sg; 643 $builderror = "^(" . $builderror . ")\$"; 644 } 645 # runerror is multiple REs separated by OR 646 if (defined $runerror) { 647 $runerror =~ s/\nOR\n/\n|/sg; 648 $runerror = "^(" . $runerror . ")\$"; 649 } 650 651 # save some results for build and run phases 652 $$CREF{"TEST_$name"} = { 653 TEST_BUILD => $buildcmd, 654 TEST_BUILD_OUTPUT => $builderror, 655 TEST_CRASHES => $crashes, 656 TEST_RUN_OUTPUT => $runerror, 657 TEST_CFLAGS => $cflags, 658 TEST_ENV => $envstring, 659 TEST_RUN => $run, 660 }; 661 662 return 1; 663} 664 665# Builds a simple test 666sub build_simple { 667 my %C = %{shift()}; 668 my $name = shift; 669 my %T = %{$C{"TEST_$name"}}; 670 chdir_verbose "$C{DIR}/$name.build"; 671 672 my $ext = $ALL_TESTS{$name}; 673 my $file = "$DIR/$name.$ext"; 674 675 if ($T{TEST_CRASHES}) { 676 `echo '$crashcatch' > crashcatch.c`; 677 make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c"); 678 die "$?" if $?; 679 } 680 681 my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE} $T{TEST_CFLAGS} $file -o $name.out"; 682 683 my $output = make($cmd); 684 685 my $ok; 686 if (my $builderror = $T{TEST_BUILD_OUTPUT}) { 687 # check for expected output and ignore $? 688 if ($output =~ /$builderror/) { 689 $ok = 1; 690 } else { 691 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 692 colorprint $red, $output; 693 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 694 print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n"; 695 $ok = 0; 696 } 697 } elsif ($?) { 698 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 699 colorprint $red, $output; 700 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 701 print "${red}FAIL: $name (build failed)$def\n"; 702 $ok = 0; 703 } elsif ($output ne "") { 704 print "${red}FAIL: /// test '$name' \\\\\\$def\n"; 705 colorprint $red, $output; 706 print "${red}FAIL: \\\\\\ test '$name' ///$def\n"; 707 print "${red}FAIL: $name (unexpected build output)$def\n"; 708 $ok = 0; 709 } else { 710 $ok = 1; 711 } 712 713 714 if ($ok) { 715 foreach my $file (glob("*.out *.dylib *.bundle")) { 716 make("dsymutil $file"); 717 } 718 } 719 720 return $ok; 721} 722 723# Run a simple test (testname.out, with error checking of stdout and stderr) 724sub run_simple { 725 my %C = %{shift()}; 726 my $name = shift; 727 my %T = %{$C{"TEST_$name"}}; 728 729 if (! $T{TEST_RUN}) { 730 print "PASS: $name (build only)\n"; 731 return 1; 732 } 733 else { 734 chdir_verbose "$C{DIR}/$name.build"; 735 } 736 737 my $env = "$C{ENV} $T{TEST_ENV}"; 738 if ($T{TEST_CRASHES}) { 739 $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib"; 740 } 741 742 my $output; 743 744 if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) { 745 # run on iOS device 746 747 my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build"; 748 my $remotedyld = " DYLD_LIBRARY_PATH=$remotedir"; 749 $remotedyld .= ":/var/root/test/" if ($C{TESTLIB} ne $TESTLIBPATH); 750 751 # elide host-specific paths 752 $env =~ s/DYLD_LIBRARY_PATH=\S+//; 753 $env =~ s/DYLD_ROOT_PATH=\S+//; 754 755 my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'"; 756 $output = make("$cmd"); 757 } 758 else { 759 # run locally 760 761 my $cmd = "env $env ./$name.out"; 762 $output = make("sh -c '$cmd 2>&1' 2>&1"); 763 # need extra sh level to capture "sh: Illegal instruction" after crash 764 # fixme fail if $? except tests that expect to crash 765 } 766 767 return check_output(\%C, $name, split("\n", $output)); 768} 769 770 771my %compiler_memo; 772sub find_compiler { 773 my ($cc, $sdk, $sdk_path) = @_; 774 775 # memoize 776 my $key = $cc . ':' . $sdk; 777 my $result = $compiler_memo{$key}; 778 return $result if defined $result; 779 780 if ($sdk eq "system") { 781 $result = `xcrun -find $cc 2>/dev/null`; 782 } else { 783 $result = `xcrun -sdk $sdk -find $cc 2>/dev/null`; 784 } 785 786 chomp $result; 787 $compiler_memo{$key} = $result; 788 return $result; 789} 790 791sub make_one_config { 792 my $configref = shift; 793 my $root = shift; 794 my %C = %{$configref}; 795 796 # Aliases 797 $C{LANGUAGE} = "objective-c" if $C{LANGUAGE} eq "objc"; 798 $C{LANGUAGE} = "objective-c++" if $C{LANGUAGE} eq "objc++"; 799 800 # Look up SDK 801 # Try exact match first. 802 # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal"). 803 my @sdks = getsdks(); 804 if ($VERBOSE) { 805 print "Installed SDKs: @sdks\n"; 806 } 807 my $exactsdk = undef; 808 my $prefixsdk = undef; 809 foreach my $sdk (@sdks) { 810 my $SDK = $C{SDK}; 811 $exactsdk = $sdk if ($sdk eq $SDK); 812 # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2" 813 $prefixsdk = $sdk if ($sdk =~ /^$SDK[0-9]/ && $sdk gt $prefixsdk); 814 } 815 if ($exactsdk) { 816 $C{SDK} = $exactsdk; 817 } elsif ($prefixsdk) { 818 $C{SDK} = $prefixsdk; 819 } else { 820 die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n"; 821 } 822 823 # set the config name now, after massaging the language and sdk, 824 # but before adding other settings 825 my $configname = config_name(%C); 826 die if ($configname =~ /'/); 827 die if ($configname =~ / /); 828 ($C{NAME} = $configname) =~ s/~/ /g; 829 (my $configdir = $configname) =~ s#/##g; 830 $C{DIR} = "$BUILDDIR/$configdir"; 831 832 $C{SDK_PATH} = "/"; 833 if ($C{SDK} ne "system") { 834 ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/); 835 } 836 837 # Look up test library (possible in root or SDK_PATH) 838 839 my $rootarg = $root; 840 my $symroot; 841 my @sympaths = ( (glob "$root/*~sym")[0], 842 (glob "$root/BuildRecords/*_install/Symbols")[0], 843 "$root/Symbols" ); 844 my @dstpaths = ( (glob "$root/*~dst")[0], 845 (glob "$root/BuildRecords/*_install/Root")[0], 846 "$root/Root" ); 847 for(my $i = 0; $i < scalar(@sympaths); $i++) { 848 if (-e $sympaths[$i] && -e $dstpaths[$i]) { 849 $symroot = $sympaths[$i]; 850 $root = $dstpaths[$i]; 851 last; 852 } 853 } 854 855 if ($root ne "" && -e "$root$C{SDK_PATH}$TESTLIBPATH") { 856 $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH"; 857 } elsif (-e "$root$TESTLIBPATH") { 858 $C{TESTLIB} = "$root$TESTLIBPATH"; 859 } elsif (-e "$root/$TESTLIBNAME") { 860 $C{TESTLIB} = "$root/$TESTLIBNAME"; 861 } else { 862 die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n" 863 # . join("\n", @dstpaths) . "\n" 864 ; 865 } 866 867 if (-e "$symroot/$TESTLIBNAME.dSYM") { 868 $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM"; 869 } 870 871 if ($VERBOSE) { 872 my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`; 873 while (my $uuid = shift @uuids) { 874 print "note: $uuid"; 875 } 876 } 877 878 # Look up compilers 879 my $cc = $C{CC}; 880 my $cxx = cplusplus($C{CC}); 881 if (! $BUILD) { 882 $C{CC} = $cc; 883 $C{CXX} = $cxx; 884 } else { 885 $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH}); 886 $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH}); 887 888 die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC}; 889 die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX}; 890 } 891 892 # Populate cflags 893 894 # save-temps so dsymutil works so debug info works 895 my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} "; 896 my $objcflags = ""; 897 898 if ($C{SDK} ne "system") { 899 $cflags .= " -isysroot '$C{SDK_PATH}'"; 900 $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'"; 901 } 902 903 if ($C{SDK} =~ /^iphoneos[0-9]/ && $cflags !~ /-mios-version-min/) { 904 my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9+])/); 905 $cflags .= " -mios-version-min=$vers"; 906 } 907 if ($C{SDK} =~ /^iphonesimulator[0-9]/ && $cflags !~ /-mios-simulator-version-min/) { 908 my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/); 909 $cflags .= " -mios-simulator-version-min=$vers"; 910 } 911 if ($C{SDK} =~ /^iphonesimulator/) { 912 $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch"; 913 } 914 915 if ($root ne "") { 916 my $library_path = dirname($C{TESTLIB}); 917 $cflags .= " -L$library_path"; 918 $cflags .= " -I '$root/usr/include'"; 919 $cflags .= " -I '$root/usr/local/include'"; 920 921 if ($C{SDK_PATH} ne "/") { 922 $cflags .= " -I '$root$C{SDK_PATH}/usr/include'"; 923 $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'"; 924 } 925 } 926 927 if ($C{CC} =~ /clang/) { 928 $cflags .= " -Qunused-arguments -fno-caret-diagnostics"; 929 $cflags .= " -stdlib=$C{STDLIB} -fno-objc-link-runtime"; 930 } 931 932 933 # Populate objcflags 934 935 $objcflags .= " -lobjc"; 936 if ($C{MEM} eq "gc") { 937 $objcflags .= " -fobjc-gc"; 938 } 939 elsif ($C{MEM} eq "arc") { 940 $objcflags .= " -fobjc-arc"; 941 } 942 elsif ($C{MEM} eq "mrc") { 943 # nothing 944 } 945 else { 946 die "unrecognized MEM '$C{MEM}'\n"; 947 } 948 949 if (supportslibauto($C{SDK})) { 950 # do this even for non-GC tests 951 $objcflags .= " -lauto"; 952 } 953 954 # Populate ENV_PREFIX 955 $C{ENV} = "LANG=C MallocScribble=1"; 956 $C{ENV} .= " VERBOSE=1" if $VERBOSE; 957 if ($root ne "") { 958 my $library_path = dirname($C{TESTLIB}); 959 die "no spaces allowed in root" if $library_path =~ /\s+/; 960 $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path" if ($library_path ne "/usr/lib"); 961 } 962 if ($C{SDK_PATH} ne "/") { 963 die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/; 964 $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}"; 965 } 966 if ($C{GUARDMALLOC}) { 967 $ENV{GUARDMALLOC} = "1"; # checked by tests and errcheck.pl 968 $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib"; 969 } 970 if ($C{SDK} =~ /^iphonesimulator[0-9]/) { 971 my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/); 972 $C{ENV} .= 973 " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" . 974 " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" . 975 " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers"; 976 } 977 978 # Populate compiler commands 979 $C{COMPILE_C} = "env LANG=C '$C{CC}' $cflags -x c -std=gnu99"; 980 $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++"; 981 $C{COMPILE_M} = "env LANG=C '$C{CC}' $cflags $objcflags -x objective-c -std=gnu99"; 982 $C{COMPILE_MM} = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++"; 983 984 $C{COMPILE} = $C{COMPILE_C} if $C{LANGUAGE} eq "c"; 985 $C{COMPILE} = $C{COMPILE_CXX} if $C{LANGUAGE} eq "c++"; 986 $C{COMPILE} = $C{COMPILE_M} if $C{LANGUAGE} eq "objective-c"; 987 $C{COMPILE} = $C{COMPILE_MM} if $C{LANGUAGE} eq "objective-c++"; 988 die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE}; 989 990 ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g; 991 ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g; 992 ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g; 993 994 995 # Reject some self-inconsistent configurations 996 if ($C{MEM} !~ /^(mrc|arc|gc)$/) { 997 die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n"; 998 } 999 1000 if ($C{MEM} eq "gc" && $C{SDK} =~ /^iphone/) { 1001 print "note: skipping configuration $C{NAME}\n"; 1002 print "note: because SDK=$C{SDK} does not support MEM=$C{MEM}\n"; 1003 return 0; 1004 } 1005 if ($C{MEM} eq "arc" && $C{SDK} !~ /^iphone/ && $C{ARCH} eq "i386") { 1006 print "note: skipping configuration $C{NAME}\n"; 1007 print "note: because 32-bit Mac does not support MEM=$C{MEM}\n"; 1008 return 0; 1009 } 1010 if ($C{MEM} eq "arc" && $C{CC} !~ /clang/) { 1011 print "note: skipping configuration $C{NAME}\n"; 1012 print "note: because CC=$C{CC} does not support MEM=$C{MEM}\n"; 1013 return 0; 1014 } 1015 1016 if ($C{STDLIB} ne "libstdc++" && $C{CC} !~ /clang/) { 1017 print "note: skipping configuration $C{NAME}\n"; 1018 print "note: because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n"; 1019 return 0; 1020 } 1021 1022 %$configref = %C; 1023} 1024 1025sub make_configs { 1026 my ($root, %args) = @_; 1027 1028 my @results = ({}); # start with one empty config 1029 1030 for my $key (keys %args) { 1031 my @newresults; 1032 my @values = @{$args{$key}}; 1033 for my $configref (@results) { 1034 my %config = %{$configref}; 1035 for my $value (@values) { 1036 my %newconfig = %config; 1037 $newconfig{$key} = $value; 1038 push @newresults, \%newconfig; 1039 } 1040 } 1041 @results = @newresults; 1042 } 1043 1044 my @newresults; 1045 for my $configref(@results) { 1046 if (make_one_config($configref, $root)) { 1047 push @newresults, $configref; 1048 } 1049 } 1050 1051 return @newresults; 1052} 1053 1054sub config_name { 1055 my %config = @_; 1056 my $name = ""; 1057 for my $key (sort keys %config) { 1058 $name .= '~' if $name ne ""; 1059 $name .= "$key=$config{$key}"; 1060 } 1061 return $name; 1062} 1063 1064sub run_one_config { 1065 my %C = %{shift()}; 1066 my @tests = @_; 1067 1068 # Build and run 1069 my $testcount = 0; 1070 my $failcount = 0; 1071 1072 my @gathertests; 1073 foreach my $test (@tests) { 1074 if ($VERBOSE) { 1075 print "\nGATHER $test\n"; 1076 } 1077 1078 if ($ALL_TESTS{$test}) { 1079 gather_simple(\%C, $test) || next; # not pass, not fail 1080 push @gathertests, $test; 1081 } else { 1082 die "No test named '$test'\n"; 1083 } 1084 } 1085 1086 my @builttests; 1087 if (!$BUILD) { 1088 @builttests = @gathertests; 1089 $testcount = scalar(@gathertests); 1090 } else { 1091 my $configdir = $C{DIR}; 1092 print $configdir, "\n" if $VERBOSE; 1093 mkdir $configdir || die; 1094 1095 foreach my $test (@gathertests) { 1096 if ($VERBOSE) { 1097 print "\nBUILD $test\n"; 1098 } 1099 mkdir "$configdir/$test.build" || die; 1100 1101 if ($ALL_TESTS{$test}) { 1102 $testcount++; 1103 if (!build_simple(\%C, $test)) { 1104 $failcount++; 1105 } else { 1106 push @builttests, $test; 1107 } 1108 } else { 1109 die "No test named '$test'\n"; 1110 } 1111 } 1112 } 1113 1114 if (!$RUN || !scalar(@builttests)) { 1115 # nothing to do 1116 } 1117 else { 1118 if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) { 1119 # upload all tests to iOS device 1120 make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/"); 1121 die "Couldn't rsync tests to device\n" if ($?); 1122 1123 # upload library to iOS device 1124 if ($C{TESTLIB} ne $TESTLIBPATH) { 1125 # hack - send thin library because device may use lib=armv7 1126 # even though app=armv6, and we want to set the lib's arch 1127 make("xcrun -sdk $C{SDK} lipo -output /tmp/$TESTLIBNAME -thin $C{ARCH} $C{TESTLIB} || cp $C{TESTLIB} /tmp/$TESTLIBNAME"); 1128 die "Couldn't thin $C{TESTLIB} to $C{ARCH}\n" if ($?); 1129 make("RSYNC_PASSWORD=alpine rsync -av /tmp/$TESTLIBNAME rsync://root\@localhost:10873/root/var/root/test/"); 1130 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?); 1131 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/"); 1132 } 1133 } 1134 1135 foreach my $test (@builttests) { 1136 print "\nRUN $test\n" if ($VERBOSE); 1137 1138 if ($ALL_TESTS{$test}) 1139 { 1140 if (!run_simple(\%C, $test)) { 1141 $failcount++; 1142 } 1143 } else { 1144 die "No test named '$test'\n"; 1145 } 1146 } 1147 } 1148 1149 return ($testcount, $failcount); 1150} 1151 1152 1153 1154# Return value if set by "$argname=value" on the command line 1155# Return $default if not set. 1156sub getargs { 1157 my ($argname, $default) = @_; 1158 1159 foreach my $arg (@ARGV) { 1160 my ($value) = ($arg =~ /^$argname=(.+)$/); 1161 return [split ',', $value] if defined $value; 1162 } 1163 1164 return [$default]; 1165} 1166 1167# Return 1 or 0 if set by "$argname=1" or "$argname=0" on the 1168# command line. Return $default if not set. 1169sub getbools { 1170 my ($argname, $default) = @_; 1171 1172 my @values = @{getargs($argname, $default)}; 1173 return [( map { ($_ eq "0") ? 0 : 1 } @values )]; 1174} 1175 1176sub getarg { 1177 my ($argname, $default) = @_; 1178 my @values = @{getargs($argname, $default)}; 1179 die "Only one value allowed for $argname\n" if @values > 1; 1180 return $values[0]; 1181} 1182 1183sub getbool { 1184 my ($argname, $default) = @_; 1185 my @values = @{getbools($argname, $default)}; 1186 die "Only one value allowed for $argname\n" if @values > 1; 1187 return $values[0]; 1188} 1189 1190 1191# main 1192my %args; 1193 1194 1195my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386"; 1196$args{ARCH} = getargs("ARCH", 0); 1197$args{ARCH} = getargs("ARCHS", $default_arch) if !@{$args{ARCH}}[0]; 1198 1199$args{SDK} = getargs("SDK", "system"); 1200 1201$args{MEM} = getargs("MEM", "mrc"); 1202$args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ]; 1203$args{STDLIB} = getargs("STDLIB", "libstdc++"); 1204 1205$args{CC} = getargs("CC", "clang"); 1206 1207$args{GUARDMALLOC} = getbools("GUARDMALLOC", 0); 1208 1209$BUILD = getbool("BUILD", 1); 1210$RUN = getbool("RUN", 1); 1211$VERBOSE = getbool("VERBOSE", 0); 1212 1213my $root = getarg("ROOT", ""); 1214$root =~ s#/*$##; 1215 1216my @tests = gettests(); 1217 1218print "note: -----\n"; 1219print "note: testing root '$root'\n"; 1220 1221my @configs = make_configs($root, %args); 1222 1223print "note: -----\n"; 1224print "note: testing ", scalar(@configs), " configurations:\n"; 1225for my $configref (@configs) { 1226 my $configname = $$configref{NAME}; 1227 print "note: configuration $configname\n"; 1228} 1229 1230if ($BUILD) { 1231 `rm -rf '$BUILDDIR'`; 1232 mkdir "$BUILDDIR" || die; 1233} 1234 1235my $failed = 0; 1236 1237my $testconfigs = @configs; 1238my $failconfigs = 0; 1239my $testcount = 0; 1240my $failcount = 0; 1241for my $configref (@configs) { 1242 my $configname = $$configref{NAME}; 1243 print "note: -----\n"; 1244 print "note: \nnote: $configname\nnote: \n"; 1245 1246 (my $t, my $f) = eval { run_one_config($configref, @tests); }; 1247 if ($@) { 1248 chomp $@; 1249 print "${red}FAIL: $configname${def}\n"; 1250 print "${red}FAIL: $@${def}\n"; 1251 $failconfigs++; 1252 } else { 1253 my $color = ($f ? $red : ""); 1254 print "note:\n"; 1255 print "${color}note: $configname$def\n"; 1256 print "${color}note: $t tests, $f failures$def\n"; 1257 $testcount += $t; 1258 $failcount += $f; 1259 $failconfigs++ if ($f); 1260 } 1261} 1262 1263print "note: -----\n"; 1264my $color = ($failconfigs ? $red : ""); 1265print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n"; 1266print "${color}note: $testcount tests, $failcount failures$def\n"; 1267 1268$failed = ($failconfigs ? 1 : 0); 1269 1270exit ($failed ? 1 : 0); 1271