ParallelTestRunner.java revision 6:5a1b0714df0e
1/* 2 * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26package jdk.nashorn.internal.test.framework; 27 28import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE; 29import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE; 30import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST; 31import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK; 32import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS; 33 34import java.io.BufferedReader; 35import java.io.ByteArrayOutputStream; 36import java.io.File; 37import java.io.FileInputStream; 38import java.io.FileOutputStream; 39import java.io.FileReader; 40import java.io.IOException; 41import java.io.InputStreamReader; 42import java.io.OutputStream; 43import java.io.PrintStream; 44import java.io.PrintWriter; 45import java.io.StringReader; 46import java.nio.file.FileSystems; 47import java.nio.file.Files; 48import java.nio.file.StandardCopyOption; 49import java.util.ArrayList; 50import java.util.Collections; 51import java.util.Comparator; 52import java.util.List; 53import java.util.Locale; 54import java.util.Map; 55import java.util.Properties; 56import java.util.Set; 57import java.util.TreeSet; 58import java.util.concurrent.Callable; 59import java.util.concurrent.CancellationException; 60import java.util.concurrent.CountDownLatch; 61import java.util.concurrent.ExecutionException; 62import java.util.concurrent.ExecutorService; 63import java.util.concurrent.Executors; 64import java.util.concurrent.Future; 65import java.util.concurrent.TimeUnit; 66import java.util.regex.Matcher; 67import java.util.regex.Pattern; 68import jdk.nashorn.internal.test.framework.TestFinder.TestFactory; 69 70/** 71 * Parallel test runner runs tests in multiple threads - but avoids any dependency 72 * on third-party test framework library such as TestNG. 73 */ 74public class ParallelTestRunner { 75 76 // ParallelTestRunner-specific 77 private static final String TEST_JS_THREADS = "test.js.threads"; 78 private static final String TEST_JS_REPORT_FILE = "test.js.report.file"; 79 private static final int THREADS = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors()); 80 81 private final List<ScriptRunnable> tests = new ArrayList<>(); 82 private final Set<String> orphans = new TreeSet<>(); 83 private final ExecutorService executor = Executors.newFixedThreadPool(THREADS); 84 85 // Ctrl-C handling 86 private final CountDownLatch finishedLatch = new CountDownLatch(1); 87 private final Thread shutdownHook = new Thread() { 88 @Override 89 public void run() { 90 if (!executor.isTerminated()) { 91 executor.shutdownNow(); 92 try { 93 executor.awaitTermination(25, TimeUnit.SECONDS); 94 finishedLatch.await(5, TimeUnit.SECONDS); 95 } catch (final InterruptedException e) { 96 // empty 97 } 98 } 99 } 100 }; 101 102 public ParallelTestRunner() throws Exception { 103 suite(); 104 } 105 106 private static PrintStream outputStream() { 107 final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, ""); 108 PrintStream output = System.out; 109 110 if (!reportFile.isEmpty()) { 111 try { 112 output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile))); 113 } catch (final IOException e) { 114 System.err.println(e); 115 } 116 } 117 118 return output; 119 } 120 121 public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> { 122 private final Result result = new Result(); 123 124 public class Result { 125 private boolean passed = true; 126 public String expected; 127 public String out; 128 public String err; 129 public Throwable exception; 130 131 public ScriptRunnable getTest() { 132 return ScriptRunnable.this; 133 } 134 135 public boolean passed() { 136 return passed; 137 } 138 139 @Override 140 public String toString() { 141 return getTest().toString(); 142 } 143 } 144 145 public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) { 146 super(framework, testFile, engineOptions, testOptions, scriptArguments); 147 } 148 149 @Override 150 protected void log(String msg) { 151 System.err.println(msg); 152 } 153 154 @Override 155 protected void fail(final String message) { 156 throw new TestFailedError(message); 157 } 158 159 @Override 160 protected void compile() throws IOException { 161 final ByteArrayOutputStream out = new ByteArrayOutputStream(); 162 final ByteArrayOutputStream err = new ByteArrayOutputStream(); 163 final List<String> args = getCompilerArgs(); 164 int errors; 165 try { 166 errors = evaluateScript(out, err, args.toArray(new String[args.size()])); 167 } catch (final AssertionError e) { 168 final PrintWriter writer = new PrintWriter(err); 169 e.printStackTrace(writer); 170 writer.flush(); 171 errors = 1; 172 } 173 if (errors != 0 || checkCompilerMsg) { 174 result.err = err.toString(); 175 if (expectCompileFailure || checkCompilerMsg) { 176 final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName())); 177 TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray()))); 178 outputDest.println("--"); 179 } 180 if (errors != 0 && !expectCompileFailure) { 181 fail(String.format("%d errors compiling %s", errors, testFile)); 182 } 183 if (checkCompilerMsg) { 184 compare(getErrorFileName(), expectedFileName, true); 185 } 186 } 187 if (expectCompileFailure && errors == 0) { 188 fail(String.format("No errors encountered compiling negative test %s", testFile)); 189 } 190 } 191 192 @Override 193 protected void execute() { 194 final List<String> args = getRuntimeArgs(); 195 final ByteArrayOutputStream out = new ByteArrayOutputStream(); 196 final ByteArrayOutputStream err = new ByteArrayOutputStream(); 197 198 try { 199 final int errors = evaluateScript(out, err, args.toArray(new String[args.size()])); 200 201 if (errors != 0 || err.size() > 0) { 202 if (expectRunFailure) { 203 return; 204 } 205 if (!ignoreStdError) { 206 207 try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) { 208 outputFile.write(out.toByteArray()); 209 errorFile.write(err.toByteArray()); 210 } 211 212 result.out = out.toString(); 213 result.err = err.toString(); 214 fail(err.toString()); 215 } 216 } 217 218 if (compare) { 219 final File expectedFile = new File(expectedFileName); 220 try { 221 BufferedReader expected; 222 if (expectedFile.exists()) { 223 expected = new BufferedReader(new FileReader(expectedFile)); 224 } else { 225 expected = new BufferedReader(new StringReader("")); 226 } 227 compare(new BufferedReader(new StringReader(out.toString())), expected, false); 228 } catch (final Throwable ex) { 229 if (expectedFile.exists()) { 230 copyExpectedFile(); 231 } 232 try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) { 233 outputFile.write(out.toByteArray()); 234 errorFile.write(err.toByteArray()); 235 } 236 throw ex; 237 } 238 } 239 } catch (final IOException e) { 240 if (!expectRunFailure) { 241 fail("Failure running test " + testFile + ": " + e.getMessage()); 242 } // else success 243 } 244 } 245 246 private void compare(final String outputFileName, final String expected, final boolean compareCompilerMsg) throws IOException { 247 final File expectedFile = new File(expected); 248 249 BufferedReader expectedReader; 250 if (expectedFile.exists()) { 251 expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName))); 252 } else { 253 expectedReader = new BufferedReader(new StringReader("")); 254 } 255 256 final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(outputFileName))); 257 258 compare(actual, expectedReader, compareCompilerMsg); 259 } 260 261 private void copyExpectedFile() { 262 if (!new File(expectedFileName).exists()) { 263 return; 264 } 265 // copy expected file overwriting existing file and preserving last 266 // modified time of source 267 try { 268 Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); 269 } catch (final IOException ex) { 270 fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage()); 271 } 272 } 273 274 @Override 275 public Result call() { 276 try { 277 runTest(); 278 } catch (final Throwable ex) { 279 result.exception = ex; 280 result.passed = false; 281 } 282 return result; 283 } 284 285 private String getOutputFileName() { 286 buildDir.mkdirs(); 287 return outputFileName; 288 } 289 290 private String getErrorFileName() { 291 buildDir.mkdirs(); 292 return errorFileName; 293 } 294 295 private String getCopyExpectedFileName() { 296 buildDir.mkdirs(); 297 return copyExpectedFileName; 298 } 299 } 300 301 private void suite() throws Exception { 302 Locale.setDefault(new Locale("")); 303 System.setOut(outputStream()); 304 305 final TestFactory<ScriptRunnable> testFactory = new TestFactory<ScriptRunnable>() { 306 @Override 307 public ScriptRunnable createTest(String framework, File testFile, List<String> engineOptions, Map<String, String> testOptions, List<String> arguments) { 308 return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments); 309 } 310 311 @Override 312 public void log(String msg) { 313 System.err.println(msg); 314 } 315 }; 316 317 TestFinder.findAllTests(tests, orphans, testFactory); 318 319 Collections.sort(tests, new Comparator<ScriptRunnable>() { 320 @Override 321 public int compare(final ScriptRunnable o1, final ScriptRunnable o2) { 322 return o1.testFile.compareTo(o2.testFile); 323 } 324 }); 325 } 326 327 public void run() { 328 final int testCount = tests.size(); 329 int passCount = 0; 330 int doneCount = 0; 331 System.out.printf("Found %d tests.\n", testCount); 332 final long startTime = System.nanoTime(); 333 334 Runtime.getRuntime().addShutdownHook(shutdownHook); 335 336 final List<Future<ScriptRunnable.Result>> futures = new ArrayList<>(); 337 for (final ScriptRunnable test : tests) { 338 futures.add(executor.submit(test)); 339 } 340 341 executor.shutdown(); 342 try { 343 executor.awaitTermination(60, TimeUnit.MINUTES); 344 } catch (final InterruptedException ex) { 345 // empty 346 } 347 348 final List<ScriptRunnable.Result> results = new ArrayList<>(); 349 for (final Future<ScriptRunnable.Result> future : futures) { 350 if (future.isDone()) { 351 try { 352 final ScriptRunnable.Result result = future.get(); 353 results.add(result); 354 doneCount++; 355 if (result.passed()) { 356 passCount++; 357 } 358 } catch (CancellationException | ExecutionException ex) { 359 ex.printStackTrace(); 360 } catch (final InterruptedException ex) { 361 assert false : "should not reach here"; 362 } 363 } 364 } 365 366 Collections.sort(results, new Comparator<ScriptRunnable.Result>() { 367 @Override 368 public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) { 369 return o1.getTest().testFile.compareTo(o2.getTest().testFile); 370 } 371 }); 372 373 boolean hasFailed = false; 374 for (final ScriptRunnable.Result result : results) { 375 if (!result.passed()) { 376 if (hasFailed == false) { 377 hasFailed = true; 378 System.out.println(); 379 System.out.println("FAILED TESTS"); 380 } 381 382 System.out.println(result.getTest()); 383 if (result.exception != null) { 384 final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString(); 385 System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n"); 386 System.out.print(result.out != null ? result.out : ""); 387 } 388 } 389 } 390 391 final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s] 392 System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60); 393 System.out.flush(); 394 395 finishedLatch.countDown(); 396 397 if (hasFailed) { 398 throw new AssertionError("TEST FAILED"); 399 } 400 } 401 402 public static void main(final String[] args) throws Exception { 403 parseArgs(args); 404 405 new ParallelTestRunner().run(); 406 } 407 408 private static void parseArgs(final String[] args) { 409 if (args.length > 0) { 410 String roots = ""; 411 String reportFile = ""; 412 for (int i = 0; i < args.length; i++) { 413 if (args[i].equals("--roots") && i != args.length - 1) { 414 roots += args[++i] + " "; 415 } else if (args[i].equals("--report-file") && i != args.length - 1) { 416 reportFile = args[++i]; 417 } else if (args[i].equals("--test262")) { 418 try { 419 setTest262Properties(); 420 } catch (final IOException ex) { 421 System.err.println(ex); 422 } 423 } 424 } 425 if (!roots.isEmpty()) { 426 System.setProperty(TEST_JS_ROOTS, roots.trim()); 427 } 428 if (!reportFile.isEmpty()) { 429 System.setProperty(TEST_JS_REPORT_FILE, reportFile); 430 } 431 } 432 } 433 434 private static void setTest262Properties() throws IOException { 435 System.setProperty(TEST_JS_ROOTS, "test/test262/test/suite/"); 436 System.setProperty(TEST_JS_FRAMEWORK, "test/script/test262.js test/test262/test/harness/framework.js test/test262/test/harness/sta.js"); 437 System.setProperty(TEST_JS_EXCLUDES_FILE, "test/test262/test/config/excludelist.xml"); 438 System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true"); 439 440 final Properties projectProperties = new Properties(); 441 projectProperties.load(new FileInputStream("project.properties")); 442 String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", ""); 443 final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}"); 444 for (;;) { 445 final Matcher matcher = pattern.matcher(excludeList); 446 if (!matcher.find()) { 447 break; 448 } 449 final String propertyValue = projectProperties.getProperty(matcher.group(1), ""); 450 excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end()); 451 } 452 System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList); 453 } 454 455 public static final class OutputStreamDelegator extends OutputStream { 456 private final OutputStream[] streams; 457 458 public OutputStreamDelegator(final OutputStream... streams) { 459 this.streams = streams; 460 } 461 462 @Override 463 public void write(final int b) throws IOException { 464 for (final OutputStream stream : streams) { 465 stream.write(b); 466 } 467 } 468 469 @Override 470 public void flush() throws IOException { 471 for (final OutputStream stream : streams) { 472 stream.flush(); 473 } 474 } 475 } 476} 477 478final class TestFailedError extends Error { 479 private static final long serialVersionUID = 1L; 480 481 public TestFailedError(final String message) { 482 super(message); 483 } 484 485 public TestFailedError(final String message, final Throwable cause) { 486 super(message, cause); 487 } 488 489 public TestFailedError(final Throwable cause) { 490 super(cause); 491 } 492} 493