1#!/usr/bin/perl
2
3# test.pl
4# Run unit tests.
5
6use strict;
7use File::Basename;
8
9chdir dirname $0;
10chomp (my $DIR = `pwd`);
11
12my $TESTLIBNAME = "libobjc.A.dylib";
13my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
14
15my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
16
17# xterm colors
18my $red = "\e[41;37m";
19my $yellow = "\e[43;37m";
20my $def = "\e[0m";
21
22# clean, help
23if (scalar(@ARGV) == 1) {
24    my $arg = $ARGV[0];
25    if ($arg eq "clean") {
26        my $cmd = "rm -rf $BUILDDIR *~";
27        print "$cmd\n";
28        `$cmd`;
29        exit 0;
30    }
31    elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
32        print(<<END);
33usage: $0 [options] [testname ...]
34       $0 clean
35       $0 help
36
37testname:
38    `testname` runs a specific test. If no testnames are given, runs all tests.
39
40options:
41    ARCH=<arch>
42    SDK=<sdk name>
43    ROOT=/path/to/project.roots/
44
45    CC=<compiler name>
46
47    LANGUAGE=c,c++,objective-c,objective-c++,swift
48    MEM=mrc,arc,gc
49    STDLIB=libc++,libstdc++
50    GUARDMALLOC=0|1
51
52    BUILD=0|1
53    RUN=0|1
54    VERBOSE=0|1
55
56examples:
57
58    test installed library, x86_64, no gc
59    $0
60
61    test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
62    $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
63
64    test buildit-built root with iOS simulator
65    $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
66
67    test buildit-built root on attached iOS device
68    $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
69END
70        exit 0;
71    }
72}
73
74#########################################################################
75## Tests
76
77my %ALL_TESTS;
78
79#########################################################################
80## Variables for use in complex build and run rules
81
82# variable         # example value
83
84# things you can multiplex on the command line
85# ARCH=i386,x86_64,armv6,armv7
86# SDK=macosx,iphoneos,iphonesimulator
87# LANGUAGE=c,c++,objective-c,objective-c++,swift
88# CC=clang,gcc-4.2,llvm-gcc-4.2
89# MEM=mrc,arc,gc
90# STDLIB=libc++,libstdc++
91# GUARDMALLOC=0,1
92
93# things you can set once on the command line
94# ROOT=/path/to/project.roots
95# BUILD=0|1
96# RUN=0|1
97# VERBOSE=0|1
98
99
100
101my $BUILD;
102my $RUN;
103my $VERBOSE;
104
105my $crashcatch = <<'END';
106// interpose-able code to catch crashes, print, and exit cleanly
107#include <signal.h>
108#include <string.h>
109#include <unistd.h>
110
111// from dyld-interposing.h
112#define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
113
114static void catchcrash(int sig)
115{
116    const char *msg;
117    switch (sig) {
118    case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
119    case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
120    case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
121    case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
122    case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
123    case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
124    default: msg = "SIG\?\?\?\?\\n"; break;
125    }
126    write(STDERR_FILENO, msg, strlen(msg));
127    _exit(0);
128}
129
130static void setupcrash(void) __attribute__((constructor));
131static void setupcrash(void)
132{
133    signal(SIGILL, &catchcrash);
134    signal(SIGBUS, &catchcrash);
135    signal(SIGSYS, &catchcrash);
136    signal(SIGSEGV, &catchcrash);
137    signal(SIGTRAP, &catchcrash);
138    signal(SIGABRT, &catchcrash);
139}
140
141
142static int hacked = 0;
143ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
144{
145    if (!hacked) {
146        setupcrash();
147        hacked = 1;
148    }
149    return write(fildes, buf, nbyte);
150}
151
152DYLD_INTERPOSE(hacked_write, write);
153
154END
155
156
157#########################################################################
158## Harness
159
160
161# map language to buildable extensions for that language
162my %extensions_for_language = (
163    "c"     => ["c"],
164    "objective-c" => ["c", "m"],
165    "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
166    "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
167    "swift" => ["swift"],
168
169    "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
170    );
171
172# map extension to languages
173my %languages_for_extension = (
174    "c" => ["c", "objective-c", "c++", "objective-c++"],
175    "m" => ["objective-c", "objective-c++"],
176    "mm" => ["objective-c++"],
177    "cc" => ["c++", "objective-c++"],
178    "cp" => ["c++", "objective-c++"],
179    "cpp" => ["c++", "objective-c++"],
180    "cxx" => ["c++", "objective-c++"],
181    "c++" => ["c++", "objective-c++"],
182    "swift" => ["swift"],
183    );
184
185# Run some newline-separated commands like `make` would, stopping if any fail
186# run("cmd1 \n cmd2 \n cmd3")
187sub make {
188    my $output = "";
189    my @cmds = split("\n", $_[0]);
190    die if scalar(@cmds) == 0;
191    $? = 0;
192    foreach my $cmd (@cmds) {
193        chomp $cmd;
194        next if $cmd =~ /^\s*$/;
195        $cmd .= " 2>&1";
196        print "$cmd\n" if $VERBOSE;
197        $output .= `$cmd`;
198        last if $?;
199    }
200    print "$output\n" if $VERBOSE;
201    return $output;
202}
203
204sub chdir_verbose {
205    my $dir = shift;
206    chdir $dir || die;
207    print "cd $dir\n" if $VERBOSE;
208}
209
210
211# Return test names from the command line.
212# Returns all tests if no tests were named.
213sub gettests {
214    my @tests;
215
216    foreach my $arg (@ARGV) {
217        push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
218    }
219
220    opendir(my $dir, $DIR) || die;
221    while (my $file = readdir($dir)) {
222        my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
223        next if ! $languages_for_extension{$ext};
224
225        open(my $in, "< $file") || die "$file";
226        my $contents = join "", <$in>;
227        if (defined $ALL_TESTS{$name}) {
228            print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n";
229        } else {
230            $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
231        }
232        close($in);
233    }
234    closedir($dir);
235
236    if (scalar(@tests) == 0) {
237        @tests = keys %ALL_TESTS;
238    }
239
240    @tests = sort @tests;
241
242    return @tests;
243}
244
245
246# Turn a C compiler name into a C++ compiler name.
247sub cplusplus {
248    my ($c) = @_;
249    if ($c =~ /cc/) {
250        $c =~ s/cc/\+\+/;
251        return $c;
252    }
253    return $c . "++";                         # e.g. clang => clang++
254}
255
256# Turn a C compiler name into a Swift compiler name
257sub swift {
258    my ($c) = @_;
259    $c =~ s#[^/]*$#swift#;
260    return $c;
261}
262
263# Returns an array of all sdks from `xcodebuild -showsdks`
264my @sdks_memo;
265sub getsdks {
266    if (!@sdks_memo) {
267        @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
268    }
269    return @sdks_memo;
270}
271
272# Returns whether the given sdk supports -lauto
273sub supportslibauto {
274    my ($sdk) = @_;
275    return 1 if $sdk =~ /^macosx/;
276    return 0 if $sdk =~ /^iphone/;
277    die;
278}
279
280# print text with a colored prefix on each line
281sub colorprint {
282    my $color = shift;
283    while (my @lines = split("\n", shift)) {
284        for my $line (@lines) {
285            chomp $line;
286            print "$color $def$line\n";
287        }
288    }
289}
290
291sub rewind {
292    seek($_[0], 0, 0);
293}
294
295# parse name=value,value pairs
296sub readconditions {
297    my ($conditionstring) = @_;
298
299    my %results;
300    my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
301    for my $condition (@conditions) {
302        my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
303        $results{$name} = [split ',', $values];
304    }
305
306    return %results;
307}
308
309# Get the name of the system SDK from sw_vers
310sub systemsdkname {
311    my @lines = `/usr/bin/sw_vers`;
312    my $name;
313    my $vers;
314    for my $line (@lines) {
315        ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
316        ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
317    }
318
319    $name =~ s/ //g;
320    $name = lc($name);
321    my $internal = "";
322    if (-d "/usr/local/include/objc") {
323        if ($name eq "macosx") {
324            $internal = "internal";
325        } else {
326            $internal = ".internal";
327        }
328    }
329    return $name . $vers . $internal;
330}
331
332sub check_output {
333    my %C = %{shift()};
334    my $name = shift;
335    my @output = @_;
336
337    my %T = %{$C{"TEST_$name"}};
338
339    # Quietly strip MallocScribble before saving the "original" output
340    # because it is distracting.
341    filter_malloc(\@output);
342
343    my @original_output = @output;
344
345    # Run result-checking passes, reducing @output each time
346    my $xit = 1;
347    my $bad = "";
348    my $warn = "";
349    my $runerror = $T{TEST_RUN_OUTPUT};
350    filter_hax(\@output);
351    filter_verbose(\@output);
352    $warn = filter_warn(\@output);
353    $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
354    $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
355    $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
356    $bad = filter_bad(\@output)  if ($bad eq "");
357
358    # OK line should be the only one left
359    $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
360
361    if ($bad ne "") {
362        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
363        colorprint($red, @original_output);
364        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
365        print "${red}FAIL: $name: $bad$def\n";
366        $xit = 0;
367    }
368    elsif ($warn ne "") {
369        print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
370        colorprint($yellow, @original_output);
371        print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
372        print "PASS: $name (with warnings)\n";
373    }
374    else {
375        print "PASS: $name\n";
376    }
377    return $xit;
378}
379
380sub filter_expected
381{
382    my $outputref = shift;
383    my %C = %{shift()};
384    my $name = shift;
385
386    my %T = %{$C{"TEST_$name"}};
387    my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
388
389    my $bad = "";
390
391    my $output = join("\n", @$outputref) . "\n";
392    if ($output !~ /$runerror/) {
393	$bad = "(run output does not match TEST_RUN_OUTPUT)";
394	@$outputref = ("FAIL: $name");
395    } else {
396	@$outputref = ("OK: $name");  # pacify later filter
397    }
398
399    return $bad;
400}
401
402sub filter_bad
403{
404    my $outputref = shift;
405    my $bad = "";
406
407    my @new_output;
408    for my $line (@$outputref) {
409	if ($line =~ /^BAD: (.*)/) {
410	    $bad = "(failed)";
411	} else {
412	    push @new_output, $line;
413	}
414    }
415
416    @$outputref = @new_output;
417    return $bad;
418}
419
420sub filter_warn
421{
422    my $outputref = shift;
423    my $warn = "";
424
425    my @new_output;
426    for my $line (@$outputref) {
427	if ($line !~ /^WARN: (.*)/) {
428	    push @new_output, $line;
429        } else {
430	    $warn = "(warned)";
431	}
432    }
433
434    @$outputref = @new_output;
435    return $warn;
436}
437
438sub filter_verbose
439{
440    my $outputref = shift;
441
442    my @new_output;
443    for my $line (@$outputref) {
444	if ($line !~ /^VERBOSE: (.*)/) {
445	    push @new_output, $line;
446	}
447    }
448
449    @$outputref = @new_output;
450}
451
452sub filter_hax
453{
454    my $outputref = shift;
455
456    my @new_output;
457    for my $line (@$outputref) {
458	if ($line !~ /Class OS_tcp_/) {
459	    push @new_output, $line;
460	}
461    }
462
463    @$outputref = @new_output;
464}
465
466sub filter_valgrind
467{
468    my $outputref = shift;
469    my $errors = 0;
470    my $leaks = 0;
471
472    my @new_output;
473    for my $line (@$outputref) {
474	if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
475	    # --track-origins warning (harmless)
476	    next;
477	}
478	if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
479	    # signals unsupported (harmless)
480	    next;
481	}
482	if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
483	    # signals unsupported (harmless)
484	    next;
485	}
486	if ($line !~ /^^\.*==\d+==/) {
487	    # not valgrind output
488	    push @new_output, $line;
489	    next;
490	}
491
492	my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
493	if (defined $errcount  &&  $errcount > 0) {
494	    $errors = 1;
495	}
496
497	(my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
498	if (defined $leakcount  &&  $leakcount > 0) {
499	    $leaks = 1;
500	}
501    }
502
503    @$outputref = @new_output;
504
505    my $bad = "";
506    $bad .= "(valgrind errors)" if ($errors);
507    $bad .= "(valgrind leaks)" if ($leaks);
508    return $bad;
509}
510
511
512
513sub filter_malloc
514{
515    my $outputref = shift;
516    my $errors = 0;
517
518    my @new_output;
519    my $count = 0;
520    for my $line (@$outputref) {
521        # Ignore MallocScribble prologue.
522        # Ignore MallocStackLogging prologue.
523        if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
524            $line =~ /Deleted objects will be dirtied by the collector/  ||
525            $line =~ /malloc: stack logs being written into/  ||
526            $line =~ /malloc: recording malloc and VM allocation stacks/)
527        {
528            next;
529	}
530
531        # not malloc output
532        push @new_output, $line;
533
534    }
535
536    @$outputref = @new_output;
537}
538
539sub filter_guardmalloc
540{
541    my $outputref = shift;
542    my $errors = 0;
543
544    my @new_output;
545    my $count = 0;
546    for my $line (@$outputref) {
547	if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
548	    # not guardmalloc output
549	    push @new_output, $line;
550	    next;
551	}
552
553        # Ignore 4 lines of guardmalloc prologue.
554        # Anything further is a guardmalloc error.
555        if (++$count > 4) {
556            $errors = 1;
557        }
558    }
559
560    @$outputref = @new_output;
561
562    my $bad = "";
563    $bad .= "(guardmalloc errors)" if ($errors);
564    return $bad;
565}
566
567# TEST_SOMETHING
568# text
569# text
570# END
571sub extract_multiline {
572    my ($flag, $contents, $name) = @_;
573    if ($contents =~ /$flag\n/) {
574        my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
575        die "$name used $flag without END\n"  if !defined($output);
576        return $output;
577    }
578    return undef;
579}
580
581
582# TEST_SOMETHING
583# text
584# OR
585# text
586# END
587sub extract_multiple_multiline {
588    my ($flag, $contents, $name) = @_;
589    if ($contents =~ /$flag\n/) {
590        my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
591        die "$name used $flag without END\n"  if !defined($output);
592
593        $output =~ s/\nOR\n/\n|/sg;
594        $output = "^(" . $output . ")\$";
595        return $output;
596    }
597    return undef;
598}
599
600
601sub gather_simple {
602    my $CREF = shift;
603    my %C = %{$CREF};
604    my $name = shift;
605    chdir_verbose $DIR;
606
607    my $ext = $ALL_TESTS{$name};
608    my $file = "$name.$ext";
609    return 0 if !$file;
610
611    # search file for 'TEST_CONFIG' or '#include "test.h"'
612    # also collect other values:
613    # TEST_CONFIG test conditions
614    # TEST_ENV environment prefix
615    # TEST_CFLAGS compile flags
616    # TEST_BUILD build instructions
617    # TEST_BUILD_OUTPUT expected build stdout/stderr
618    # TEST_RUN_OUTPUT expected run stdout/stderr
619    open(my $in, "< $file") || die;
620    my $contents = join "", <$in>;
621
622    my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
623    my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
624    my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
625    my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
626    my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
627    my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
628    my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
629    my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
630    my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
631
632    return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
633
634    if ($disabled) {
635        print "${yellow}SKIP: $name    (disabled by TEST_DISABLED)$def\n";
636        return 0;
637    }
638
639    # check test conditions
640
641    my $run = 1;
642    my %conditions = readconditions($conditionstring);
643    if (! $conditions{LANGUAGE}) {
644        # implicit language restriction from file extension
645        $conditions{LANGUAGE} = $languages_for_extension{$ext};
646    }
647    for my $condkey (keys %conditions) {
648        my @condvalues = @{$conditions{$condkey}};
649
650        # special case: RUN=0 does not affect build
651        if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
652            $run = 0;
653            next;
654        }
655
656        my $testvalue = $C{$condkey};
657        next if !defined($testvalue);
658        # testvalue is the configuration being run now
659        # condvalues are the allowed values for this test
660
661        my $ok = 0;
662        for my $condvalue (@condvalues) {
663
664            # special case: objc and objc++
665            if ($condkey eq "LANGUAGE") {
666                $condvalue = "objective-c" if $condvalue eq "objc";
667                $condvalue = "objective-c++" if $condvalue eq "objc++";
668            }
669
670            $ok = 1  if ($testvalue eq $condvalue);
671
672            # special case: SDK allows prefixes
673            if ($condkey eq "SDK") {
674                $ok = 1  if ($testvalue =~ /^$condvalue/);
675            }
676
677            # special case: CC and CXX allow substring matches
678            if ($condkey eq "CC"  ||  $condkey eq "CXX") {
679                $ok = 1  if ($testvalue =~ /$condvalue/);
680            }
681
682            last if $ok;
683        }
684
685        if (!$ok) {
686            my $plural = (@condvalues > 1) ? "one of: " : "";
687            print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
688            return 0;
689        }
690    }
691
692    # save some results for build and run phases
693    $$CREF{"TEST_$name"} = {
694        TEST_BUILD => $buildcmd,
695        TEST_BUILD_OUTPUT => $builderror,
696        TEST_CRASHES => $crashes,
697        TEST_RUN_OUTPUT => $runerror,
698        TEST_CFLAGS => $cflags,
699        TEST_ENV => $envstring,
700        TEST_RUN => $run,
701    };
702
703    return 1;
704}
705
706# Builds a simple test
707sub build_simple {
708    my %C = %{shift()};
709    my $name = shift;
710    my %T = %{$C{"TEST_$name"}};
711    chdir_verbose "$C{DIR}/$name.build";
712
713    my $ext = $ALL_TESTS{$name};
714    my $file = "$DIR/$name.$ext";
715
716    if ($T{TEST_CRASHES}) {
717        `echo '$crashcatch' > crashcatch.c`;
718        make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
719        die "$?" if $?;
720    }
721
722    my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
723
724    my $output = make($cmd);
725
726    # rdar://10163155
727    $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
728
729    my $ok;
730    if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
731        # check for expected output and ignore $?
732        if ($output =~ /$builderror/) {
733            $ok = 1;
734        } else {
735            print "${red}FAIL: /// test '$name' \\\\\\$def\n";
736            colorprint $red, $output;
737            print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
738            print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
739            $ok = 0;
740        }
741    } elsif ($?) {
742        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
743        colorprint $red, $output;
744        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
745        print "${red}FAIL: $name (build failed)$def\n";
746        $ok = 0;
747    } elsif ($output ne "") {
748        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
749        colorprint $red, $output;
750        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
751        print "${red}FAIL: $name (unexpected build output)$def\n";
752        $ok = 0;
753    } else {
754        $ok = 1;
755    }
756
757
758    if ($ok) {
759        foreach my $file (glob("*.out *.dylib *.bundle")) {
760            make("dsymutil $file");
761        }
762    }
763
764    return $ok;
765}
766
767# Run a simple test (testname.out, with error checking of stdout and stderr)
768sub run_simple {
769    my %C = %{shift()};
770    my $name = shift;
771    my %T = %{$C{"TEST_$name"}};
772
773    if (! $T{TEST_RUN}) {
774        print "PASS: $name (build only)\n";
775        return 1;
776    }
777    else {
778        chdir_verbose "$C{DIR}/$name.build";
779    }
780
781    my $env = "$C{ENV} $T{TEST_ENV}";
782    if ($T{TEST_CRASHES}) {
783        $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
784    }
785
786    my $output;
787
788    if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
789        # run on iOS device
790
791        my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
792        my $remotedyld = "";
793        $remotedyld .= " DYLD_LIBRARY_PATH=$remotedir";
794        $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
795
796        # elide host-specific paths
797        $env =~ s/DYLD_LIBRARY_PATH=\S+//;
798        $env =~ s/DYLD_ROOT_PATH=\S+//;
799
800        my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'";
801        $output = make("$cmd");
802    }
803    else {
804        # run locally
805
806        my $cmd = "env $env ./$name.out";
807        $output = make("sh -c '$cmd 2>&1' 2>&1");
808        # need extra sh level to capture "sh: Illegal instruction" after crash
809        # fixme fail if $? except tests that expect to crash
810    }
811
812    return check_output(\%C, $name, split("\n", $output));
813}
814
815
816my %compiler_memo;
817sub find_compiler {
818    my ($cc, $sdk, $sdk_path) = @_;
819
820    # memoize
821    my $key = $cc . ':' . $sdk;
822    my $result = $compiler_memo{$key};
823    return $result if defined $result;
824
825    $result  = `xcrun -sdk $sdk -find $cc 2>/dev/null`;
826
827    chomp $result;
828    $compiler_memo{$key} = $result;
829    return $result;
830}
831
832sub make_one_config {
833    my $configref = shift;
834    my $root = shift;
835    my %C = %{$configref};
836
837    # Aliases
838    $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
839    $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
840
841    # Look up SDK
842    # Try exact match first.
843    # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
844    my @sdks = getsdks();
845    if ($VERBOSE) {
846        print "Installed SDKs: @sdks\n";
847    }
848    my $exactsdk = undef;
849    my $prefixsdk = undef;
850    foreach my $sdk (@sdks) {
851        my $SDK = $C{SDK};
852        $exactsdk = $sdk  if ($sdk eq $SDK);
853        # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
854        $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
855    }
856    if ($exactsdk) {
857        $C{SDK} = $exactsdk;
858    } elsif ($prefixsdk) {
859        $C{SDK} = $prefixsdk;
860    } else {
861        die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
862    }
863
864    # set the config name now, after massaging the language and sdk,
865    # but before adding other settings
866    my $configname = config_name(%C);
867    die if ($configname =~ /'/);
868    die if ($configname =~ / /);
869    ($C{NAME} = $configname) =~ s/~/ /g;
870    (my $configdir = $configname) =~ s#/##g;
871    $C{DIR} = "$BUILDDIR/$configdir";
872
873    ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
874
875    # Look up test library (possible in root or SDK_PATH)
876
877    my $rootarg = $root;
878    my $symroot;
879    my @sympaths = ( (glob "$root/*~sym")[0],
880                     (glob "$root/BuildRecords/*_install/Symbols")[0],
881                     "$root/Symbols" );
882    my @dstpaths = ( (glob "$root/*~dst")[0],
883                     (glob "$root/BuildRecords/*_install/Root")[0],
884                     "$root/Root" );
885    for(my $i = 0; $i < scalar(@sympaths); $i++) {
886        if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
887            $symroot = $sympaths[$i];
888            $root = $dstpaths[$i];
889            last;
890        }
891    }
892
893    if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
894        $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
895    } elsif (-e "$root$TESTLIBPATH") {
896        $C{TESTLIB} = "$root$TESTLIBPATH";
897    } elsif (-e "$root/$TESTLIBNAME") {
898        $C{TESTLIB} = "$root/$TESTLIBNAME";
899    } else {
900        die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n"
901            # . join("\n", @dstpaths) . "\n"
902            ;
903    }
904
905    if (-e "$symroot/$TESTLIBNAME.dSYM") {
906        $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM";
907    }
908
909    if ($VERBOSE) {
910        my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
911        while (my $uuid = shift @uuids) {
912            print "note: $uuid";
913        }
914    }
915
916    # Look up compilers
917    my $cc = $C{CC};
918    my $cxx = cplusplus($C{CC});
919    my $swift = swift($C{CC});
920    if (! $BUILD) {
921        $C{CC} = $cc;
922        $C{CXX} = $cxx;
923        $C{SWIFT} = $swift
924    } else {
925        $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH});
926        $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH});
927        $C{SWIFT} = find_compiler($swift, $C{SDK}, $C{SDK_PATH});
928
929        die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC};
930        die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX};
931        die "No compiler '$swift' ('$C{SWIFT}') in SDK '$C{SDK}'\n" if !-e $C{SWIFT};
932    }
933
934    # Populate cflags
935
936    # save-temps so dsymutil works so debug info works
937    my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
938    my $objcflags = "";
939    my $swiftflags = "-g ";
940
941    $cflags .= " -isysroot '$C{SDK_PATH}'";
942    $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
943    $swiftflags .= " -sdk '$C{SDK_PATH}'";
944
945    my $target = "";
946    if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-mios-version-min/) {
947        my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9]+)/);
948        $cflags .= " -mios-version-min=$vers";
949        $target = "$C{ARCH}-apple-ios$vers";
950    }
951    elsif ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-mios-simulator-version-min/) {
952        my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9]+)/);
953        $cflags .= " -mios-simulator-version-min=$vers";
954        $target = "$C{ARCH}-apple-ios$vers";
955    }
956    else {
957        my ($vers) = ($C{SDK} =~ /^macosx([0-9]+\.[0-9]+)/);
958        $vers = ""  if !defined($vers);
959        $target = "$C{ARCH}-apple-macosx$vers";
960    }
961    $swiftflags .= " -target $target";
962
963    if ($C{SDK} =~ /^iphonesimulator/  &&  $C{ARCH} eq "i386") {
964        $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
965    }
966
967    if ($root ne "") {
968        my $library_path = dirname($C{TESTLIB});
969        $cflags .= " -L$library_path";
970        $cflags .= " -I '$root/usr/include'";
971        $cflags .= " -I '$root/usr/local/include'";
972
973        if ($C{SDK_PATH} ne "/") {
974            $cflags .= " -I '$root$C{SDK_PATH}/usr/include'";
975            $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'";
976        }
977    }
978
979    if ($C{CC} =~ /clang/) {
980        $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
981        $cflags .= " -stdlib=$C{STDLIB}"; # fixme -fno-objc-link-runtime"
982        $cflags .= " -Wl,-segalign,0x4000 ";
983    }
984
985
986    # Populate objcflags
987
988    $objcflags .= " -lobjc";
989    if ($C{MEM} eq "gc") {
990        $objcflags .= " -fobjc-gc";
991    }
992    elsif ($C{MEM} eq "arc") {
993        $objcflags .= " -fobjc-arc";
994    }
995    elsif ($C{MEM} eq "mrc") {
996        # nothing
997    }
998    else {
999        die "unrecognized MEM '$C{MEM}'\n";
1000    }
1001
1002    if (supportslibauto($C{SDK})) {
1003        # do this even for non-GC tests
1004        $objcflags .= " -lauto";
1005    }
1006
1007    # Populate ENV_PREFIX
1008    $C{ENV} = "LANG=C MallocScribble=1";
1009    $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
1010    if ($root ne "") {
1011        my $library_path = dirname($C{TESTLIB});
1012        die "no spaces allowed in root" if $library_path =~ /\s+/;
1013        $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
1014    }
1015    if ($C{SDK_PATH} ne "/") {
1016        die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
1017        $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
1018    }
1019    if ($C{GUARDMALLOC}) {
1020        $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
1021        $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1022    }
1023    if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
1024        my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
1025        $C{ENV} .=
1026            " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
1027            " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
1028            " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
1029    }
1030
1031    # Populate compiler commands
1032    $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
1033    $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
1034    $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1035    $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
1036    $C{COMPILE_SWIFT} = "env LANG=C '$C{SWIFT}' $swiftflags";
1037
1038    $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1039    $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1040    $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1041    $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1042    $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1043    die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1044
1045    ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
1046    ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1047    ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1048
1049
1050    # Reject some self-inconsistent configurations
1051    if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
1052        die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
1053    }
1054
1055    if ($C{MEM} eq "gc"  &&  $C{SDK} =~ /^iphone/) {
1056        print "note: skipping configuration $C{NAME}\n";
1057        print "note:   because SDK=$C{SDK} does not support MEM=$C{MEM}\n";
1058        return 0;
1059    }
1060    if ($C{MEM} eq "arc"  &&  $C{SDK} !~ /^iphone/  &&  $C{ARCH} eq "i386") {
1061        print "note: skipping configuration $C{NAME}\n";
1062        print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
1063        return 0;
1064    }
1065    if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1066        print "note: skipping configuration $C{NAME}\n";
1067        print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1068        return 0;
1069    }
1070
1071    if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
1072        print "note: skipping configuration $C{NAME}\n";
1073        print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
1074        return 0;
1075    }
1076
1077    # fixme
1078    if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1079        print "note: skipping configuration $C{NAME}\n";
1080        print "note:   because ARCH=$C{ARCH} does not support LANGAUGE=SWIFT\n";
1081        return 0;
1082    }
1083
1084    %$configref = %C;
1085}
1086
1087sub make_configs {
1088    my ($root, %args) = @_;
1089
1090    my @results = ({});  # start with one empty config
1091
1092    for my $key (keys %args) {
1093        my @newresults;
1094        my @values = @{$args{$key}};
1095        for my $configref (@results) {
1096            my %config = %{$configref};
1097            for my $value (@values) {
1098                my %newconfig = %config;
1099                $newconfig{$key} = $value;
1100                push @newresults, \%newconfig;
1101            }
1102        }
1103        @results = @newresults;
1104    }
1105
1106    my @newresults;
1107    for my $configref(@results) {
1108        if (make_one_config($configref, $root)) {
1109            push @newresults, $configref;
1110        }
1111    }
1112
1113    return @newresults;
1114}
1115
1116sub config_name {
1117    my %config = @_;
1118    my $name = "";
1119    for my $key (sort keys %config) {
1120        $name .= '~'  if $name ne "";
1121        $name .= "$key=$config{$key}";
1122    }
1123    return $name;
1124}
1125
1126sub run_one_config {
1127    my %C = %{shift()};
1128    my @tests = @_;
1129
1130    # Build and run
1131    my $testcount = 0;
1132    my $failcount = 0;
1133
1134    my @gathertests;
1135    foreach my $test (@tests) {
1136        if ($VERBOSE) {
1137            print "\nGATHER $test\n";
1138        }
1139
1140        if ($ALL_TESTS{$test}) {
1141            gather_simple(\%C, $test) || next;  # not pass, not fail
1142            push @gathertests, $test;
1143        } else {
1144            die "No test named '$test'\n";
1145        }
1146    }
1147
1148    my @builttests;
1149    if (!$BUILD) {
1150        @builttests = @gathertests;
1151        $testcount = scalar(@gathertests);
1152    } else {
1153        my $configdir = $C{DIR};
1154        print $configdir, "\n"  if $VERBOSE;
1155        mkdir $configdir  || die;
1156
1157        foreach my $test (@gathertests) {
1158            if ($VERBOSE) {
1159                print "\nBUILD $test\n";
1160            }
1161            mkdir "$configdir/$test.build"  || die;
1162
1163            if ($ALL_TESTS{$test}) {
1164                $testcount++;
1165                if (!build_simple(\%C, $test)) {
1166                    $failcount++;
1167                } else {
1168                    push @builttests, $test;
1169                }
1170            } else {
1171                die "No test named '$test'\n";
1172            }
1173        }
1174    }
1175
1176    if (!$RUN  ||  !scalar(@builttests)) {
1177        # nothing to do
1178    }
1179    else {
1180        if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1181            # upload all tests to iOS device
1182            make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1183            die "Couldn't rsync tests to device\n" if ($?);
1184
1185            # upload library to iOS device
1186            if ($C{TESTLIB} ne $TESTLIBPATH) {
1187                make("RSYNC_PASSWORD=alpine rsync -av $C{TESTLIB} rsync://root\@localhost:10873/root/var/root/test/");
1188                die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1189                make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/");
1190            }
1191        }
1192
1193        foreach my $test (@builttests) {
1194            print "\nRUN $test\n"  if ($VERBOSE);
1195
1196            if ($ALL_TESTS{$test})
1197            {
1198                if (!run_simple(\%C, $test)) {
1199                    $failcount++;
1200                }
1201            } else {
1202                die "No test named '$test'\n";
1203            }
1204        }
1205    }
1206
1207    return ($testcount, $failcount);
1208}
1209
1210
1211
1212# Return value if set by "$argname=value" on the command line
1213# Return $default if not set.
1214sub getargs {
1215    my ($argname, $default) = @_;
1216
1217    foreach my $arg (@ARGV) {
1218        my ($value) = ($arg =~ /^$argname=(.+)$/);
1219        return [split ',', $value] if defined $value;
1220    }
1221
1222    return [split ',', $default];
1223}
1224
1225# Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1226# command line. Return $default if not set.
1227sub getbools {
1228    my ($argname, $default) = @_;
1229
1230    my @values = @{getargs($argname, $default)};
1231    return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1232}
1233
1234sub getarg {
1235    my ($argname, $default) = @_;
1236    my @values = @{getargs($argname, $default)};
1237    die "Only one value allowed for $argname\n"  if @values > 1;
1238    return $values[0];
1239}
1240
1241sub getbool {
1242    my ($argname, $default) = @_;
1243    my @values = @{getbools($argname, $default)};
1244    die "Only one value allowed for $argname\n"  if @values > 1;
1245    return $values[0];
1246}
1247
1248
1249# main
1250my %args;
1251
1252
1253my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1254$args{ARCH} = getargs("ARCH", 0);
1255$args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1256
1257$args{SDK} = getargs("SDK", "macosx");
1258
1259$args{MEM} = getargs("MEM", "mrc");
1260$args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c,swift")} ];
1261$args{STDLIB} = getargs("STDLIB", "libstdc++");
1262
1263$args{CC} = getargs("CC", "clang");
1264
1265$args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1266
1267$BUILD = getbool("BUILD", 1);
1268$RUN = getbool("RUN", 1);
1269$VERBOSE = getbool("VERBOSE", 0);
1270
1271my $root = getarg("ROOT", "");
1272$root =~ s#/*$##;
1273
1274my @tests = gettests();
1275
1276print "note: -----\n";
1277print "note: testing root '$root'\n";
1278
1279my @configs = make_configs($root, %args);
1280
1281print "note: -----\n";
1282print "note: testing ", scalar(@configs), " configurations:\n";
1283for my $configref (@configs) {
1284    my $configname = $$configref{NAME};
1285    print "note: configuration $configname\n";
1286}
1287
1288if ($BUILD) {
1289    `rm -rf '$BUILDDIR'`;
1290    mkdir "$BUILDDIR" || die;
1291}
1292
1293my $failed = 0;
1294
1295my $testconfigs = @configs;
1296my $failconfigs = 0;
1297my $testcount = 0;
1298my $failcount = 0;
1299for my $configref (@configs) {
1300    my $configname = $$configref{NAME};
1301    print "note: -----\n";
1302    print "note: \nnote: $configname\nnote: \n";
1303
1304    (my $t, my $f) = eval { run_one_config($configref, @tests); };
1305    if ($@) {
1306        chomp $@;
1307        print "${red}FAIL: $configname${def}\n";
1308        print "${red}FAIL: $@${def}\n";
1309        $failconfigs++;
1310    } else {
1311        my $color = ($f ? $red : "");
1312        print "note:\n";
1313        print "${color}note: $configname$def\n";
1314        print "${color}note: $t tests, $f failures$def\n";
1315        $testcount += $t;
1316        $failcount += $f;
1317        $failconfigs++ if ($f);
1318    }
1319}
1320
1321print "note: -----\n";
1322my $color = ($failconfigs ? $red : "");
1323print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1324print "${color}note: $testcount tests, $failcount failures$def\n";
1325
1326$failed = ($failconfigs ? 1 : 0);
1327
1328exit ($failed ? 1 : 0);
1329