1#!/usr/bin/perl
2
3# test.pl
4# Run unit tests.
5
6use strict;
7use File::Basename;
8
9chdir dirname $0;
10chomp (my $DIR = `pwd`);
11
12my $TESTLIBNAME = "libobjc.A.dylib";
13my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
14
15my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
16
17# xterm colors
18my $red = "\e[41;37m";
19my $yellow = "\e[43;37m";
20my $def = "\e[0m";
21
22# clean, help
23if (scalar(@ARGV) == 1) {
24    my $arg = $ARGV[0];
25    if ($arg eq "clean") {
26        my $cmd = "rm -rf $BUILDDIR *~";
27        print "$cmd\n";
28        `$cmd`;
29        exit 0;
30    }
31    elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
32        print(<<END);
33usage: $0 [options] [testname ...]
34       $0 clean
35       $0 help
36
37testname:
38    `testname` runs a specific test. If no testnames are given, runs all tests.
39
40options:
41    ARCH=<arch>
42    SDK=<sdk name>
43    ROOT=/path/to/project.roots/
44
45    CC=<compiler name>
46
47    MEM=mrc,arc,gc
48    STDLIB=libc++,libstdc++
49    GUARDMALLOC=0|1
50
51    BUILD=0|1
52    RUN=0|1
53    VERBOSE=0|1
54
55examples:
56
57    test installed library, x86_64, no gc
58    $0
59
60    test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
61    $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
62
63    test buildit-built root with iOS simulator
64    $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
65
66    test buildit-built root on attached iOS device
67    $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
68END
69        exit 0;
70    }
71}
72
73#########################################################################
74## Tests
75
76my %ALL_TESTS;
77
78#########################################################################
79## Variables for use in complex build and run rules
80
81# variable         # example value
82
83# things you can multiplex on the command line
84# ARCH=i386,x86_64,armv6,armv7
85# SDK=system,macosx,iphoneos,iphonesimulator
86# LANGUAGE=c,c++,objective-c,objective-c++
87# CC=clang,gcc-4.2,llvm-gcc-4.2
88# MEM=mrc,arc,gc
89# STDLIB=libc++,libstdc++
90# GUARDMALLOC=0,1
91
92# things you can set once on the command line
93# ROOT=/path/to/project.roots
94# BUILD=0|1
95# RUN=0|1
96# VERBOSE=0|1
97
98
99
100my $BUILD;
101my $RUN;
102my $VERBOSE;
103
104my $crashcatch = <<'END';
105// interpose-able code to catch crashes, print, and exit cleanly
106#include <signal.h>
107#include <string.h>
108#include <unistd.h>
109
110// from dyld-interposing.h
111#define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
112
113static void catchcrash(int sig)
114{
115    const char *msg;
116    switch (sig) {
117    case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
118    case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
119    case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
120    case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
121    case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
122    case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
123    default: msg = "SIG\?\?\?\?\\n"; break;
124    }
125    write(STDERR_FILENO, msg, strlen(msg));
126    _exit(0);
127}
128
129static void setupcrash(void) __attribute__((constructor));
130static void setupcrash(void)
131{
132    signal(SIGILL, &catchcrash);
133    signal(SIGBUS, &catchcrash);
134    signal(SIGSYS, &catchcrash);
135    signal(SIGSEGV, &catchcrash);
136    signal(SIGTRAP, &catchcrash);
137    signal(SIGABRT, &catchcrash);
138}
139
140
141static int hacked = 0;
142ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
143{
144    if (!hacked) {
145        setupcrash();
146        hacked = 1;
147    }
148    return write(fildes, buf, nbyte);
149}
150
151DYLD_INTERPOSE(hacked_write, write);
152
153END
154
155
156#########################################################################
157## Harness
158
159
160# map language to buildable extensions for that language
161my %extensions_for_language = (
162    "c"     => ["c"],
163    "objective-c" => ["c", "m"],
164    "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
165    "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
166
167    "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
168    );
169
170# map extension to languages
171my %languages_for_extension = (
172    "c" => ["c", "objective-c", "c++", "objective-c++"],
173    "m" => ["objective-c", "objective-c++"],
174    "mm" => ["objective-c++"],
175    "cc" => ["c++", "objective-c++"],
176    "cp" => ["c++", "objective-c++"],
177    "cpp" => ["c++", "objective-c++"],
178    "cxx" => ["c++", "objective-c++"],
179    "c++" => ["c++", "objective-c++"],
180    );
181
182# Run some newline-separated commands like `make` would, stopping if any fail
183# run("cmd1 \n cmd2 \n cmd3")
184sub make {
185    my $output = "";
186    my @cmds = split("\n", $_[0]);
187    die if scalar(@cmds) == 0;
188    $? = 0;
189    foreach my $cmd (@cmds) {
190        chomp $cmd;
191        next if $cmd =~ /^\s*$/;
192        $cmd .= " 2>&1";
193        print "$cmd\n" if $VERBOSE;
194        $output .= `$cmd`;
195        last if $?;
196    }
197    print "$output\n" if $VERBOSE;
198    return $output;
199}
200
201sub chdir_verbose {
202    my $dir = shift;
203    chdir $dir || die;
204    print "cd $dir\n" if $VERBOSE;
205}
206
207
208# Return test names from the command line.
209# Returns all tests if no tests were named.
210sub gettests {
211    my @tests;
212
213    foreach my $arg (@ARGV) {
214        push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
215    }
216
217    opendir(my $dir, $DIR) || die;
218    while (my $file = readdir($dir)) {
219        my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
220        next if ! $languages_for_extension{$ext};
221
222        open(my $in, "< $file") || die "$file";
223        my $contents = join "", <$in>;
224        if (defined $ALL_TESTS{$name}) {
225            print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n";
226        } else {
227            $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
228        }
229        close($in);
230    }
231    closedir($dir);
232
233    if (scalar(@tests) == 0) {
234        @tests = keys %ALL_TESTS;
235    }
236
237    @tests = sort @tests;
238
239    return @tests;
240}
241
242
243# Turn a C compiler name into a C++ compiler name.
244sub cplusplus {
245    my ($c) = @_;
246    if ($c =~ /cc/) {
247        $c =~ s/cc/\+\+/;
248        return $c;
249    }
250    return $c . "++";                         # e.g. clang => clang++
251}
252
253# Returns an array of all sdks from `xcodebuild -showsdks`
254my @sdks_memo;
255sub getsdks {
256    if (!@sdks_memo) {
257        @sdks_memo = ("system", `xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
258    }
259    return @sdks_memo;
260}
261
262# Returns whether the given sdk supports -lauto
263sub supportslibauto {
264    my ($sdk) = @_;
265    return 1 if $sdk eq "system";
266    return 1 if $sdk =~ /^macosx/;
267    return 0 if $sdk =~ /^iphone/;
268    die;
269}
270
271# print text with a colored prefix on each line
272sub colorprint {
273    my $color = shift;
274    while (my @lines = split("\n", shift)) {
275        for my $line (@lines) {
276            chomp $line;
277            print "$color $def$line\n";
278        }
279    }
280}
281
282sub rewind {
283    seek($_[0], 0, 0);
284}
285
286# parse name=value,value pairs
287sub readconditions {
288    my ($conditionstring) = @_;
289
290    my %results;
291    my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
292    for my $condition (@conditions) {
293        my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
294        $results{$name} = [split ',', $values];
295    }
296
297    return %results;
298}
299
300# Get the name of the system SDK from sw_vers
301sub systemsdkname {
302    my @lines = `/usr/bin/sw_vers`;
303    my $name;
304    my $vers;
305    for my $line (@lines) {
306        ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
307        ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
308    }
309
310    $name =~ s/ //g;
311    $name = lc($name);
312    my $internal = "";
313    if (-d "/usr/local/include/objc") {
314        if ($name eq "macosx") {
315            $internal = "internal";
316        } else {
317            $internal = ".internal";
318        }
319    }
320    return $name . $vers . $internal;
321}
322
323sub check_output {
324    my %C = %{shift()};
325    my $name = shift;
326    my @output = @_;
327
328    my %T = %{$C{"TEST_$name"}};
329
330    # Quietly strip MallocScribble before saving the "original" output
331    # because it is distracting.
332    filter_malloc(\@output);
333
334    my @original_output = @output;
335
336    # Run result-checking passes, reducing @output each time
337    my $xit = 1;
338    my $bad = "";
339    my $warn = "";
340    my $runerror = $T{TEST_RUN_OUTPUT};
341    filter_verbose(\@output);
342    $warn = filter_warn(\@output);
343    $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
344    $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
345    $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
346    $bad = filter_bad(\@output)  if ($bad eq "");
347
348    # OK line should be the only one left
349    $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
350
351    if ($bad ne "") {
352        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
353        colorprint($red, @original_output);
354        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
355        print "${red}FAIL: $name: $bad$def\n";
356        $xit = 0;
357    }
358    elsif ($warn ne "") {
359        print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
360        colorprint($yellow, @original_output);
361        print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
362        print "PASS: $name (with warnings)\n";
363    }
364    else {
365        print "PASS: $name\n";
366    }
367    return $xit;
368}
369
370sub filter_expected
371{
372    my $outputref = shift;
373    my %C = %{shift()};
374    my $name = shift;
375
376    my %T = %{$C{"TEST_$name"}};
377    my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
378
379    my $bad = "";
380
381    my $output = join("\n", @$outputref) . "\n";
382    if ($output !~ /$runerror/) {
383	$bad = "(run output does not match TEST_RUN_OUTPUT)";
384	@$outputref = ("FAIL: $name");
385    } else {
386	@$outputref = ("OK: $name");  # pacify later filter
387    }
388
389    return $bad;
390}
391
392sub filter_bad
393{
394    my $outputref = shift;
395    my $bad = "";
396
397    my @new_output;
398    for my $line (@$outputref) {
399	if ($line =~ /^BAD: (.*)/) {
400	    $bad = "(failed)";
401	} else {
402	    push @new_output, $line;
403	}
404    }
405
406    @$outputref = @new_output;
407    return $bad;
408}
409
410sub filter_warn
411{
412    my $outputref = shift;
413    my $warn = "";
414
415    my @new_output;
416    for my $line (@$outputref) {
417	if ($line !~ /^WARN: (.*)/) {
418	    push @new_output, $line;
419        } else {
420	    $warn = "(warned)";
421	}
422    }
423
424    @$outputref = @new_output;
425    return $warn;
426}
427
428sub filter_verbose
429{
430    my $outputref = shift;
431
432    my @new_output;
433    for my $line (@$outputref) {
434	if ($line !~ /^VERBOSE: (.*)/) {
435	    push @new_output, $line;
436	}
437    }
438
439    @$outputref = @new_output;
440}
441
442sub filter_valgrind
443{
444    my $outputref = shift;
445    my $errors = 0;
446    my $leaks = 0;
447
448    my @new_output;
449    for my $line (@$outputref) {
450	if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
451	    # --track-origins warning (harmless)
452	    next;
453	}
454	if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
455	    # signals unsupported (harmless)
456	    next;
457	}
458	if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
459	    # signals unsupported (harmless)
460	    next;
461	}
462	if ($line !~ /^^\.*==\d+==/) {
463	    # not valgrind output
464	    push @new_output, $line;
465	    next;
466	}
467
468	my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
469	if (defined $errcount  &&  $errcount > 0) {
470	    $errors = 1;
471	}
472
473	(my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
474	if (defined $leakcount  &&  $leakcount > 0) {
475	    $leaks = 1;
476	}
477    }
478
479    @$outputref = @new_output;
480
481    my $bad = "";
482    $bad .= "(valgrind errors)" if ($errors);
483    $bad .= "(valgrind leaks)" if ($leaks);
484    return $bad;
485}
486
487
488
489sub filter_malloc
490{
491    my $outputref = shift;
492    my $errors = 0;
493
494    my @new_output;
495    my $count = 0;
496    for my $line (@$outputref) {
497        # Ignore MallocScribble prologue.
498        # Ignore MallocStackLogging prologue.
499        if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
500            $line =~ /Deleted objects will be dirtied by the collector/  ||
501            $line =~ /malloc: stack logs being written into/  ||
502            $line =~ /malloc: recording malloc and VM allocation stacks/)
503        {
504            next;
505	}
506
507        # not malloc output
508        push @new_output, $line;
509
510    }
511
512    @$outputref = @new_output;
513}
514
515sub filter_guardmalloc
516{
517    my $outputref = shift;
518    my $errors = 0;
519
520    my @new_output;
521    my $count = 0;
522    for my $line (@$outputref) {
523	if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
524	    # not guardmalloc output
525	    push @new_output, $line;
526	    next;
527	}
528
529        # Ignore 4 lines of guardmalloc prologue.
530        # Anything further is a guardmalloc error.
531        if (++$count > 4) {
532            $errors = 1;
533        }
534    }
535
536    @$outputref = @new_output;
537
538    my $bad = "";
539    $bad .= "(guardmalloc errors)" if ($errors);
540    return $bad;
541}
542
543sub gather_simple {
544    my $CREF = shift;
545    my %C = %{$CREF};
546    my $name = shift;
547    chdir_verbose $DIR;
548
549    my $ext = $ALL_TESTS{$name};
550    my $file = "$name.$ext";
551    return 0 if !$file;
552
553    # search file for 'TEST_CONFIG' or '#include "test.h"'
554    # also collect other values:
555    # TEST_CONFIG test conditions
556    # TEST_ENV environment prefix
557    # TEST_CFLAGS compile flags
558    # TEST_BUILD build instructions
559    # TEST_BUILD_OUTPUT expected build stdout/stderr
560    # TEST_RUN_OUTPUT expected run stdout/stderr
561    open(my $in, "< $file") || die;
562    my $contents = join "", <$in>;
563
564    my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
565    my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
566    my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
567    my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
568    my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
569    my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
570    my ($buildcmd) = ($contents =~ /TEST_BUILD\n(.*?\n)END[ *\/]*\n/s);
571    my ($builderror) = ($contents =~ /TEST_BUILD_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
572    my ($runerror) = ($contents =~ /TEST_RUN_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
573
574    return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
575
576    if ($disabled) {
577        print "${yellow}SKIP: $name    (disabled by TEST_DISABLED)$def\n";
578        return 0;
579    }
580
581    # check test conditions
582
583    my $run = 1;
584    my %conditions = readconditions($conditionstring);
585    if (! $conditions{LANGUAGE}) {
586        # implicit language restriction from file extension
587        $conditions{LANGUAGE} = $languages_for_extension{$ext};
588    }
589    for my $condkey (keys %conditions) {
590        my @condvalues = @{$conditions{$condkey}};
591
592        # special case: RUN=0 does not affect build
593        if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
594            $run = 0;
595            next;
596        }
597
598        my $testvalue = $C{$condkey};
599        next if !defined($testvalue);
600        # testvalue is the configuration being run now
601        # condvalues are the allowed values for this test
602
603        # special case: look up the name of SDK "system"
604        if ($condkey eq "SDK"  &&  $testvalue eq "system") {
605            $testvalue = systemsdkname();
606        }
607
608        my $ok = 0;
609        for my $condvalue (@condvalues) {
610
611            # special case: objc and objc++
612            if ($condkey eq "LANGUAGE") {
613                $condvalue = "objective-c" if $condvalue eq "objc";
614                $condvalue = "objective-c++" if $condvalue eq "objc++";
615            }
616
617            $ok = 1  if ($testvalue eq $condvalue);
618
619            # special case: SDK allows prefixes, and "system" is "macosx"
620            if ($condkey eq "SDK") {
621                $ok = 1  if ($testvalue =~ /^$condvalue/);
622                $ok = 1  if ($testvalue eq "system"  &&  "macosx" =~ /^$condvalue/);
623            }
624
625            # special case: CC and CXX allow substring matches
626            if ($condkey eq "CC"  ||  $condkey eq "CXX") {
627                $ok = 1  if ($testvalue =~ /$condvalue/);
628            }
629
630            last if $ok;
631        }
632
633        if (!$ok) {
634            my $plural = (@condvalues > 1) ? "one of: " : "";
635            print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
636            return 0;
637        }
638    }
639
640    # builderror is multiple REs separated by OR
641    if (defined $builderror) {
642        $builderror =~ s/\nOR\n/\n|/sg;
643        $builderror = "^(" . $builderror . ")\$";
644    }
645    # runerror is multiple REs separated by OR
646    if (defined $runerror) {
647        $runerror =~ s/\nOR\n/\n|/sg;
648        $runerror = "^(" . $runerror . ")\$";
649    }
650
651    # save some results for build and run phases
652    $$CREF{"TEST_$name"} = {
653        TEST_BUILD => $buildcmd,
654        TEST_BUILD_OUTPUT => $builderror,
655        TEST_CRASHES => $crashes,
656        TEST_RUN_OUTPUT => $runerror,
657        TEST_CFLAGS => $cflags,
658        TEST_ENV => $envstring,
659        TEST_RUN => $run,
660    };
661
662    return 1;
663}
664
665# Builds a simple test
666sub build_simple {
667    my %C = %{shift()};
668    my $name = shift;
669    my %T = %{$C{"TEST_$name"}};
670    chdir_verbose "$C{DIR}/$name.build";
671
672    my $ext = $ALL_TESTS{$name};
673    my $file = "$DIR/$name.$ext";
674
675    if ($T{TEST_CRASHES}) {
676        `echo '$crashcatch' > crashcatch.c`;
677        make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
678        die "$?" if $?;
679    }
680
681    my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
682
683    my $output = make($cmd);
684
685    my $ok;
686    if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
687        # check for expected output and ignore $?
688        if ($output =~ /$builderror/) {
689            $ok = 1;
690        } else {
691            print "${red}FAIL: /// test '$name' \\\\\\$def\n";
692            colorprint $red, $output;
693            print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
694            print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
695            $ok = 0;
696        }
697    } elsif ($?) {
698        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
699        colorprint $red, $output;
700        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
701        print "${red}FAIL: $name (build failed)$def\n";
702        $ok = 0;
703    } elsif ($output ne "") {
704        print "${red}FAIL: /// test '$name' \\\\\\$def\n";
705        colorprint $red, $output;
706        print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
707        print "${red}FAIL: $name (unexpected build output)$def\n";
708        $ok = 0;
709    } else {
710        $ok = 1;
711    }
712
713
714    if ($ok) {
715        foreach my $file (glob("*.out *.dylib *.bundle")) {
716            make("dsymutil $file");
717        }
718    }
719
720    return $ok;
721}
722
723# Run a simple test (testname.out, with error checking of stdout and stderr)
724sub run_simple {
725    my %C = %{shift()};
726    my $name = shift;
727    my %T = %{$C{"TEST_$name"}};
728
729    if (! $T{TEST_RUN}) {
730        print "PASS: $name (build only)\n";
731        return 1;
732    }
733    else {
734        chdir_verbose "$C{DIR}/$name.build";
735    }
736
737    my $env = "$C{ENV} $T{TEST_ENV}";
738    if ($T{TEST_CRASHES}) {
739        $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
740    }
741
742    my $output;
743
744    if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
745        # run on iOS device
746
747        my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
748        my $remotedyld = " DYLD_LIBRARY_PATH=$remotedir";
749        $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
750
751        # elide host-specific paths
752        $env =~ s/DYLD_LIBRARY_PATH=\S+//;
753        $env =~ s/DYLD_ROOT_PATH=\S+//;
754
755        my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'";
756        $output = make("$cmd");
757    }
758    else {
759        # run locally
760
761        my $cmd = "env $env ./$name.out";
762        $output = make("sh -c '$cmd 2>&1' 2>&1");
763        # need extra sh level to capture "sh: Illegal instruction" after crash
764        # fixme fail if $? except tests that expect to crash
765    }
766
767    return check_output(\%C, $name, split("\n", $output));
768}
769
770
771my %compiler_memo;
772sub find_compiler {
773    my ($cc, $sdk, $sdk_path) = @_;
774
775    # memoize
776    my $key = $cc . ':' . $sdk;
777    my $result = $compiler_memo{$key};
778    return $result if defined $result;
779
780    if ($sdk eq "system") {
781        $result  = `xcrun -find $cc 2>/dev/null`;
782    } else {
783        $result  = `xcrun -sdk $sdk -find $cc 2>/dev/null`;
784    }
785
786    chomp $result;
787    $compiler_memo{$key} = $result;
788    return $result;
789}
790
791sub make_one_config {
792    my $configref = shift;
793    my $root = shift;
794    my %C = %{$configref};
795
796    # Aliases
797    $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
798    $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
799
800    # Look up SDK
801    # Try exact match first.
802    # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
803    my @sdks = getsdks();
804    if ($VERBOSE) {
805        print "Installed SDKs: @sdks\n";
806    }
807    my $exactsdk = undef;
808    my $prefixsdk = undef;
809    foreach my $sdk (@sdks) {
810        my $SDK = $C{SDK};
811        $exactsdk = $sdk  if ($sdk eq $SDK);
812        # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
813        $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
814    }
815    if ($exactsdk) {
816        $C{SDK} = $exactsdk;
817    } elsif ($prefixsdk) {
818        $C{SDK} = $prefixsdk;
819    } else {
820        die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
821    }
822
823    # set the config name now, after massaging the language and sdk,
824    # but before adding other settings
825    my $configname = config_name(%C);
826    die if ($configname =~ /'/);
827    die if ($configname =~ / /);
828    ($C{NAME} = $configname) =~ s/~/ /g;
829    (my $configdir = $configname) =~ s#/##g;
830    $C{DIR} = "$BUILDDIR/$configdir";
831
832    $C{SDK_PATH} = "/";
833    if ($C{SDK} ne "system") {
834        ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
835    }
836
837    # Look up test library (possible in root or SDK_PATH)
838
839    my $rootarg = $root;
840    my $symroot;
841    my @sympaths = ( (glob "$root/*~sym")[0],
842                     (glob "$root/BuildRecords/*_install/Symbols")[0],
843                     "$root/Symbols" );
844    my @dstpaths = ( (glob "$root/*~dst")[0],
845                     (glob "$root/BuildRecords/*_install/Root")[0],
846                     "$root/Root" );
847    for(my $i = 0; $i < scalar(@sympaths); $i++) {
848        if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
849            $symroot = $sympaths[$i];
850            $root = $dstpaths[$i];
851            last;
852        }
853    }
854
855    if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
856        $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
857    } elsif (-e "$root$TESTLIBPATH") {
858        $C{TESTLIB} = "$root$TESTLIBPATH";
859    } elsif (-e "$root/$TESTLIBNAME") {
860        $C{TESTLIB} = "$root/$TESTLIBNAME";
861    } else {
862        die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n"
863            # . join("\n", @dstpaths) . "\n"
864            ;
865    }
866
867    if (-e "$symroot/$TESTLIBNAME.dSYM") {
868        $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM";
869    }
870
871    if ($VERBOSE) {
872        my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
873        while (my $uuid = shift @uuids) {
874            print "note: $uuid";
875        }
876    }
877
878    # Look up compilers
879    my $cc = $C{CC};
880    my $cxx = cplusplus($C{CC});
881    if (! $BUILD) {
882        $C{CC} = $cc;
883        $C{CXX} = $cxx;
884    } else {
885        $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH});
886        $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH});
887
888        die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC};
889        die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX};
890    }
891
892    # Populate cflags
893
894    # save-temps so dsymutil works so debug info works
895    my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
896    my $objcflags = "";
897
898    if ($C{SDK} ne "system") {
899        $cflags .= " -isysroot '$C{SDK_PATH}'";
900        $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
901    }
902
903    if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-mios-version-min/) {
904        my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9+])/);
905        $cflags .= " -mios-version-min=$vers";
906    }
907    if ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-mios-simulator-version-min/) {
908        my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
909        $cflags .= " -mios-simulator-version-min=$vers";
910    }
911    if ($C{SDK} =~ /^iphonesimulator/) {
912        $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
913    }
914
915    if ($root ne "") {
916        my $library_path = dirname($C{TESTLIB});
917        $cflags .= " -L$library_path";
918        $cflags .= " -I '$root/usr/include'";
919        $cflags .= " -I '$root/usr/local/include'";
920
921        if ($C{SDK_PATH} ne "/") {
922            $cflags .= " -I '$root$C{SDK_PATH}/usr/include'";
923            $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'";
924        }
925    }
926
927    if ($C{CC} =~ /clang/) {
928        $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
929        $cflags .= " -stdlib=$C{STDLIB} -fno-objc-link-runtime";
930    }
931
932
933    # Populate objcflags
934
935    $objcflags .= " -lobjc";
936    if ($C{MEM} eq "gc") {
937        $objcflags .= " -fobjc-gc";
938    }
939    elsif ($C{MEM} eq "arc") {
940        $objcflags .= " -fobjc-arc";
941    }
942    elsif ($C{MEM} eq "mrc") {
943        # nothing
944    }
945    else {
946        die "unrecognized MEM '$C{MEM}'\n";
947    }
948
949    if (supportslibauto($C{SDK})) {
950        # do this even for non-GC tests
951        $objcflags .= " -lauto";
952    }
953
954    # Populate ENV_PREFIX
955    $C{ENV} = "LANG=C MallocScribble=1";
956    $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
957    if ($root ne "") {
958        my $library_path = dirname($C{TESTLIB});
959        die "no spaces allowed in root" if $library_path =~ /\s+/;
960        $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
961    }
962    if ($C{SDK_PATH} ne "/") {
963        die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
964        $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
965    }
966    if ($C{GUARDMALLOC}) {
967        $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
968        $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
969    }
970    if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
971        my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
972        $C{ENV} .=
973            " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
974            " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
975            " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
976    }
977
978    # Populate compiler commands
979    $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
980    $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
981    $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
982    $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
983
984    $C{COMPILE} = $C{COMPILE_C}    if $C{LANGUAGE} eq "c";
985    $C{COMPILE} = $C{COMPILE_CXX}  if $C{LANGUAGE} eq "c++";
986    $C{COMPILE} = $C{COMPILE_M}    if $C{LANGUAGE} eq "objective-c";
987    $C{COMPILE} = $C{COMPILE_MM}   if $C{LANGUAGE} eq "objective-c++";
988    die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
989
990    ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
991    ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
992    ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
993
994
995    # Reject some self-inconsistent configurations
996    if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
997        die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
998    }
999
1000    if ($C{MEM} eq "gc"  &&  $C{SDK} =~ /^iphone/) {
1001        print "note: skipping configuration $C{NAME}\n";
1002        print "note:   because SDK=$C{SDK} does not support MEM=$C{MEM}\n";
1003        return 0;
1004    }
1005    if ($C{MEM} eq "arc"  &&  $C{SDK} !~ /^iphone/  &&  $C{ARCH} eq "i386") {
1006        print "note: skipping configuration $C{NAME}\n";
1007        print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
1008        return 0;
1009    }
1010    if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1011        print "note: skipping configuration $C{NAME}\n";
1012        print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1013        return 0;
1014    }
1015
1016    if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
1017        print "note: skipping configuration $C{NAME}\n";
1018        print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
1019        return 0;
1020    }
1021
1022    %$configref = %C;
1023}
1024
1025sub make_configs {
1026    my ($root, %args) = @_;
1027
1028    my @results = ({});  # start with one empty config
1029
1030    for my $key (keys %args) {
1031        my @newresults;
1032        my @values = @{$args{$key}};
1033        for my $configref (@results) {
1034            my %config = %{$configref};
1035            for my $value (@values) {
1036                my %newconfig = %config;
1037                $newconfig{$key} = $value;
1038                push @newresults, \%newconfig;
1039            }
1040        }
1041        @results = @newresults;
1042    }
1043
1044    my @newresults;
1045    for my $configref(@results) {
1046        if (make_one_config($configref, $root)) {
1047            push @newresults, $configref;
1048        }
1049    }
1050
1051    return @newresults;
1052}
1053
1054sub config_name {
1055    my %config = @_;
1056    my $name = "";
1057    for my $key (sort keys %config) {
1058        $name .= '~'  if $name ne "";
1059        $name .= "$key=$config{$key}";
1060    }
1061    return $name;
1062}
1063
1064sub run_one_config {
1065    my %C = %{shift()};
1066    my @tests = @_;
1067
1068    # Build and run
1069    my $testcount = 0;
1070    my $failcount = 0;
1071
1072    my @gathertests;
1073    foreach my $test (@tests) {
1074        if ($VERBOSE) {
1075            print "\nGATHER $test\n";
1076        }
1077
1078        if ($ALL_TESTS{$test}) {
1079            gather_simple(\%C, $test) || next;  # not pass, not fail
1080            push @gathertests, $test;
1081        } else {
1082            die "No test named '$test'\n";
1083        }
1084    }
1085
1086    my @builttests;
1087    if (!$BUILD) {
1088        @builttests = @gathertests;
1089        $testcount = scalar(@gathertests);
1090    } else {
1091        my $configdir = $C{DIR};
1092        print $configdir, "\n"  if $VERBOSE;
1093        mkdir $configdir  || die;
1094
1095        foreach my $test (@gathertests) {
1096            if ($VERBOSE) {
1097                print "\nBUILD $test\n";
1098            }
1099            mkdir "$configdir/$test.build"  || die;
1100
1101            if ($ALL_TESTS{$test}) {
1102                $testcount++;
1103                if (!build_simple(\%C, $test)) {
1104                    $failcount++;
1105                } else {
1106                    push @builttests, $test;
1107                }
1108            } else {
1109                die "No test named '$test'\n";
1110            }
1111        }
1112    }
1113
1114    if (!$RUN  ||  !scalar(@builttests)) {
1115        # nothing to do
1116    }
1117    else {
1118        if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1119            # upload all tests to iOS device
1120            make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1121            die "Couldn't rsync tests to device\n" if ($?);
1122
1123            # upload library to iOS device
1124            if ($C{TESTLIB} ne $TESTLIBPATH) {
1125                # hack - send thin library because device may use lib=armv7
1126                # even though app=armv6, and we want to set the lib's arch
1127                make("xcrun -sdk $C{SDK} lipo -output /tmp/$TESTLIBNAME -thin $C{ARCH} $C{TESTLIB}  ||  cp $C{TESTLIB} /tmp/$TESTLIBNAME");
1128                die "Couldn't thin $C{TESTLIB} to $C{ARCH}\n" if ($?);
1129                make("RSYNC_PASSWORD=alpine rsync -av /tmp/$TESTLIBNAME rsync://root\@localhost:10873/root/var/root/test/");
1130                die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1131                make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/");
1132            }
1133        }
1134
1135        foreach my $test (@builttests) {
1136            print "\nRUN $test\n"  if ($VERBOSE);
1137
1138            if ($ALL_TESTS{$test})
1139            {
1140                if (!run_simple(\%C, $test)) {
1141                    $failcount++;
1142                }
1143            } else {
1144                die "No test named '$test'\n";
1145            }
1146        }
1147    }
1148
1149    return ($testcount, $failcount);
1150}
1151
1152
1153
1154# Return value if set by "$argname=value" on the command line
1155# Return $default if not set.
1156sub getargs {
1157    my ($argname, $default) = @_;
1158
1159    foreach my $arg (@ARGV) {
1160        my ($value) = ($arg =~ /^$argname=(.+)$/);
1161        return [split ',', $value] if defined $value;
1162    }
1163
1164    return [$default];
1165}
1166
1167# Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1168# command line. Return $default if not set.
1169sub getbools {
1170    my ($argname, $default) = @_;
1171
1172    my @values = @{getargs($argname, $default)};
1173    return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1174}
1175
1176sub getarg {
1177    my ($argname, $default) = @_;
1178    my @values = @{getargs($argname, $default)};
1179    die "Only one value allowed for $argname\n"  if @values > 1;
1180    return $values[0];
1181}
1182
1183sub getbool {
1184    my ($argname, $default) = @_;
1185    my @values = @{getbools($argname, $default)};
1186    die "Only one value allowed for $argname\n"  if @values > 1;
1187    return $values[0];
1188}
1189
1190
1191# main
1192my %args;
1193
1194
1195my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1196$args{ARCH} = getargs("ARCH", 0);
1197$args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1198
1199$args{SDK} = getargs("SDK", "system");
1200
1201$args{MEM} = getargs("MEM", "mrc");
1202$args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ];
1203$args{STDLIB} = getargs("STDLIB", "libstdc++");
1204
1205$args{CC} = getargs("CC", "clang");
1206
1207$args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1208
1209$BUILD = getbool("BUILD", 1);
1210$RUN = getbool("RUN", 1);
1211$VERBOSE = getbool("VERBOSE", 0);
1212
1213my $root = getarg("ROOT", "");
1214$root =~ s#/*$##;
1215
1216my @tests = gettests();
1217
1218print "note: -----\n";
1219print "note: testing root '$root'\n";
1220
1221my @configs = make_configs($root, %args);
1222
1223print "note: -----\n";
1224print "note: testing ", scalar(@configs), " configurations:\n";
1225for my $configref (@configs) {
1226    my $configname = $$configref{NAME};
1227    print "note: configuration $configname\n";
1228}
1229
1230if ($BUILD) {
1231    `rm -rf '$BUILDDIR'`;
1232    mkdir "$BUILDDIR" || die;
1233}
1234
1235my $failed = 0;
1236
1237my $testconfigs = @configs;
1238my $failconfigs = 0;
1239my $testcount = 0;
1240my $failcount = 0;
1241for my $configref (@configs) {
1242    my $configname = $$configref{NAME};
1243    print "note: -----\n";
1244    print "note: \nnote: $configname\nnote: \n";
1245
1246    (my $t, my $f) = eval { run_one_config($configref, @tests); };
1247    if ($@) {
1248        chomp $@;
1249        print "${red}FAIL: $configname${def}\n";
1250        print "${red}FAIL: $@${def}\n";
1251        $failconfigs++;
1252    } else {
1253        my $color = ($f ? $red : "");
1254        print "note:\n";
1255        print "${color}note: $configname$def\n";
1256        print "${color}note: $t tests, $f failures$def\n";
1257        $testcount += $t;
1258        $failcount += $f;
1259        $failconfigs++ if ($f);
1260    }
1261}
1262
1263print "note: -----\n";
1264my $color = ($failconfigs ? $red : "");
1265print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1266print "${color}note: $testcount tests, $failcount failures$def\n";
1267
1268$failed = ($failconfigs ? 1 : 0);
1269
1270exit ($failed ? 1 : 0);
1271