1#
2# ----------------------------------------------------------------------------------------------------
3#
4# Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
5# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6#
7# This code is free software; you can redistribute it and/or modify it
8# under the terms of the GNU General Public License version 2 only, as
9# published by the Free Software Foundation.
10#
11# This code is distributed in the hope that it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14# version 2 for more details (a copy is included in the LICENSE file that
15# accompanied this code).
16#
17# You should have received a copy of the GNU General Public License version
18# 2 along with this work; if not, write to the Free Software Foundation,
19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22# or visit www.oracle.com if you need additional information or have any
23# questions.
24#
25# ----------------------------------------------------------------------------------------------------
26
27import sanitycheck
28import itertools
29import json
30
31import mx
32import mx_graal
33
34def _run_benchmark(args, availableBenchmarks, runBenchmark):
35
36    vmOpts, benchmarksAndOptions = mx.extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
37
38    if availableBenchmarks is None:
39        harnessArgs = benchmarksAndOptions
40        return runBenchmark(None, harnessArgs, vmOpts)
41
42    if len(benchmarksAndOptions) == 0:
43        mx.abort('at least one benchmark name or "all" must be specified')
44    benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions))
45    harnessArgs = benchmarksAndOptions[len(benchmarks):]
46
47    if 'all' in benchmarks:
48        benchmarks = availableBenchmarks
49    else:
50        for bm in benchmarks:
51            if bm not in availableBenchmarks:
52                mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks))
53
54    failed = []
55    for bm in benchmarks:
56        if not runBenchmark(bm, harnessArgs, vmOpts):
57            failed.append(bm)
58
59    if len(failed) != 0:
60        mx.abort('Benchmark failures: ' + str(failed))
61
62def deoptalot(args):
63    """bootstrap a VM with DeoptimizeALot and VerifyOops on
64
65    If the first argument is a number, the process will be repeated
66    this number of times. All other arguments are passed to the VM."""
67    count = 1
68    if len(args) > 0 and args[0].isdigit():
69        count = int(args[0])
70        del args[0]
71
72    for _ in range(count):
73        if not mx_graal.run_vm(['-XX:-TieredCompilation', '-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version']) == 0:
74            mx.abort("Failed")
75
76def longtests(args):
77
78    deoptalot(['15', '-Xmx48m'])
79
80    dacapo(['100', 'eclipse', '-esa'])
81
82def dacapo(args):
83    """run one or more DaCapo benchmarks"""
84
85    def launcher(bm, harnessArgs, extraVmOpts):
86        return sanitycheck.getDacapo(bm, harnessArgs).test(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
87
88    _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher)
89
90def scaladacapo(args):
91    """run one or more Scala DaCapo benchmarks"""
92
93    def launcher(bm, harnessArgs, extraVmOpts):
94        return sanitycheck.getScalaDacapo(bm, harnessArgs).test(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
95
96    _run_benchmark(args, sanitycheck.dacapoScalaSanityWarmup.keys(), launcher)
97
98
99"""
100Extra benchmarks to run from 'bench()'.
101"""
102extraBenchmarks = []
103
104def bench(args):
105    """run benchmarks and parse their output for results
106
107    Results are JSON formated : {group : {benchmark : score}}."""
108    resultFile = None
109    if '-resultfile' in args:
110        index = args.index('-resultfile')
111        if index + 1 < len(args):
112            resultFile = args[index + 1]
113            del args[index]
114            del args[index]
115        else:
116            mx.abort('-resultfile must be followed by a file name')
117    resultFileCSV = None
118    if '-resultfilecsv' in args:
119        index = args.index('-resultfilecsv')
120        if index + 1 < len(args):
121            resultFileCSV = args[index + 1]
122            del args[index]
123            del args[index]
124        else:
125            mx.abort('-resultfilecsv must be followed by a file name')
126    vm = mx_graal.get_vm()
127    if len(args) is 0:
128        args = ['all']
129
130    vmArgs = [arg for arg in args if arg.startswith('-')]
131
132    def benchmarks_in_group(group):
133        prefix = group + ':'
134        return [a[len(prefix):] for a in args if a.startswith(prefix)]
135
136    results = {}
137    benchmarks = []
138    # DaCapo
139    if 'dacapo' in args or 'all' in args:
140        benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
141    else:
142        dacapos = benchmarks_in_group('dacapo')
143        for dacapo in dacapos:
144            if dacapo not in sanitycheck.dacapoSanityWarmup.keys():
145                mx.abort('Unknown DaCapo : ' + dacapo)
146            iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark]
147            if iterations > 0:
148                benchmarks += [sanitycheck.getDacapo(dacapo, ['-n', str(iterations)])]
149
150    if 'scaladacapo' in args or 'all' in args:
151        benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
152    else:
153        scaladacapos = benchmarks_in_group('scaladacapo')
154        for scaladacapo in scaladacapos:
155            if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():
156                mx.abort('Unknown Scala DaCapo : ' + scaladacapo)
157            iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark]
158            if iterations > 0:
159                benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])]
160
161    # Bootstrap
162    if 'bootstrap' in args or 'all' in args:
163        benchmarks += sanitycheck.getBootstraps()
164    # SPECjvm2008
165    if 'specjvm2008' in args or 'all' in args:
166        benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])]
167    else:
168        specjvms = benchmarks_in_group('specjvm2008')
169        for specjvm in specjvms:
170            benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])]
171
172    if 'specjbb2005' in args or 'all' in args:
173        benchmarks += [sanitycheck.getSPECjbb2005()]
174
175    if 'specjbb2013' in args:  # or 'all' in args //currently not in default set
176        benchmarks += [sanitycheck.getSPECjbb2013()]
177
178    if 'ctw-full' in args:
179        benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))
180    if 'ctw-noinline' in args:
181        benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))
182
183    for f in extraBenchmarks:
184        f(args, vm, benchmarks)
185
186    for test in benchmarks:
187        for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():
188            group = results.setdefault(groupName, {})
189            group.update(res)
190    mx.log(json.dumps(results))
191    if resultFile:
192        with open(resultFile, 'w') as f:
193            f.write(json.dumps(results))
194    if resultFileCSV:
195        with open(resultFileCSV, 'w') as f:
196            for key1, value1 in results.iteritems():
197                f.write('%s;\n' % (str(key1)))
198                for key2, value2 in sorted(value1.iteritems()):
199                    f.write('%s; %s;\n' % (str(key2), str(value2)))
200
201def specjvm2008(args):
202    """run one or more SPECjvm2008 benchmarks"""
203
204    def launcher(bm, harnessArgs, extraVmOpts):
205        return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
206
207    availableBenchmarks = set(sanitycheck.specjvm2008Names)
208    if "all" not in args:
209        # only add benchmark groups if we are not running "all"
210        for name in sanitycheck.specjvm2008Names:
211            parts = name.rsplit('.', 1)
212            if len(parts) > 1:
213                assert len(parts) == 2
214                group = parts[0]
215                availableBenchmarks.add(group)
216
217    _run_benchmark(args, sorted(availableBenchmarks), launcher)
218
219def specjbb2013(args):
220    """run the composite SPECjbb2013 benchmark"""
221
222    def launcher(bm, harnessArgs, extraVmOpts):
223        assert bm is None
224        return sanitycheck.getSPECjbb2013(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
225
226    _run_benchmark(args, None, launcher)
227
228def specjbb2015(args):
229    """run the composite SPECjbb2015 benchmark"""
230
231    def launcher(bm, harnessArgs, extraVmOpts):
232        assert bm is None
233        return sanitycheck.getSPECjbb2015(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
234
235    _run_benchmark(args, None, launcher)
236
237def specjbb2005(args):
238    """run the composite SPECjbb2005 benchmark"""
239
240    def launcher(bm, harnessArgs, extraVmOpts):
241        assert bm is None
242        return sanitycheck.getSPECjbb2005(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)
243
244    _run_benchmark(args, None, launcher)
245
246mx.update_commands(mx.suite('graal'), {
247    'dacapo': [dacapo, '[VM options] benchmarks...|"all" [DaCapo options]'],
248    'scaladacapo': [scaladacapo, '[VM options] benchmarks...|"all" [Scala DaCapo options]'],
249    'specjvm2008': [specjvm2008, '[VM options] benchmarks...|"all" [SPECjvm2008 options]'],
250    'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],
251    'specjbb2015': [specjbb2015, '[VM options] [-- [SPECjbb2015 options]]'],
252    'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
253    'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
254    'deoptalot' : [deoptalot, '[n]'],
255    'longtests' : [longtests, ''],
256})
257