JavacState.java revision 3191:e9a7033b2546
1/* 2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26package com.sun.tools.sjavac; 27 28import java.io.BufferedReader; 29import java.io.File; 30import java.io.FileNotFoundException; 31import java.io.FileReader; 32import java.io.FileWriter; 33import java.io.IOException; 34import java.io.Writer; 35import java.net.URI; 36import java.nio.file.NoSuchFileException; 37import java.text.SimpleDateFormat; 38import java.util.Collection; 39import java.util.Collections; 40import java.util.Date; 41import java.util.HashMap; 42import java.util.HashSet; 43import java.util.List; 44import java.util.Map; 45import java.util.Set; 46import java.util.stream.Collectors; 47 48import com.sun.tools.sjavac.comp.CompilationService; 49import com.sun.tools.sjavac.options.Options; 50import com.sun.tools.sjavac.pubapi.PubApi; 51 52/** 53 * The javac state class maintains the previous (prev) and the current (now) 54 * build states and everything else that goes into the javac_state file. 55 * 56 * <p><b>This is NOT part of any supported API. 57 * If you write code that depends on this, you do so at your own risk. 58 * This code and its internal interfaces are subject to change or 59 * deletion without notice.</b> 60 */ 61public class JavacState { 62 // The arguments to the compile. If not identical, then it cannot 63 // be an incremental build! 64 String theArgs; 65 // The number of cores limits how many threads are used for heavy concurrent work. 66 int numCores; 67 68 // The bin_dir/javac_state 69 private File javacState; 70 71 // The previous build state is loaded from javac_state 72 private BuildState prev; 73 // The current build state is constructed during the build, 74 // then saved as the new javac_state. 75 private BuildState now; 76 77 // Something has changed in the javac_state. It needs to be saved! 78 private boolean needsSaving; 79 // If this is a new javac_state file, then do not print unnecessary messages. 80 private boolean newJavacState; 81 82 // These are packages where something has changed and the package 83 // needs to be recompiled. Actions that trigger recompilation: 84 // * source belonging to the package has changed 85 // * artifact belonging to the package is lost, or its timestamp has been changed. 86 // * an unknown artifact has appeared, we simply delete it, but we also trigger a recompilation. 87 // * a package that is tainted, taints all packages that depend on it. 88 private Set<String> taintedPackages; 89 // After a compile, the pubapis are compared with the pubapis stored in the javac state file. 90 // Any packages where the pubapi differ are added to this set. 91 // Later we use this set and the dependency information to taint dependent packages. 92 private Set<String> packagesWithChangedPublicApis; 93 // When a module-info.java file is changed, taint the module, 94 // then taint all modules that depend on that that module. 95 // A module dependency can occur directly through a require, or 96 // indirectly through a module that does a public export for the first tainted module. 97 // When all modules are tainted, then taint all packages belonging to these modules. 98 // Then rebuild. It is perhaps possible (and valuable?) to do a more finegrained examination of the 99 // change in module-info.java, but that will have to wait. 100 private Set<String> taintedModules; 101 // The set of all packages that has been recompiled. 102 // Copy over the javac_state for the packages that did not need recompilation, 103 // verbatim from the previous (prev) to the new (now) build state. 104 private Set<String> recompiledPackages; 105 106 // The output directories filled with tasty artifacts. 107 private File binDir, gensrcDir, headerDir, stateDir; 108 109 // The current status of the file system. 110 private Set<File> binArtifacts; 111 private Set<File> gensrcArtifacts; 112 private Set<File> headerArtifacts; 113 114 // The status of the sources. 115 Set<Source> removedSources = null; 116 Set<Source> addedSources = null; 117 Set<Source> modifiedSources = null; 118 119 // Visible sources for linking. These are the only 120 // ones that -sourcepath is allowed to see. 121 Set<URI> visibleSrcs; 122 123 // Setup transform that always exist. 124 private CompileJavaPackages compileJavaPackages = new CompileJavaPackages(); 125 126 // Where to send stdout and stderr. 127 private Writer out, err; 128 129 // Command line options. 130 private Options options; 131 132 JavacState(Options op, boolean removeJavacState, Writer o, Writer e) { 133 options = op; 134 out = o; 135 err = e; 136 numCores = options.getNumCores(); 137 theArgs = options.getStateArgsString(); 138 binDir = Util.pathToFile(options.getDestDir()); 139 gensrcDir = Util.pathToFile(options.getGenSrcDir()); 140 headerDir = Util.pathToFile(options.getHeaderDir()); 141 stateDir = Util.pathToFile(options.getStateDir()); 142 javacState = new File(stateDir, "javac_state"); 143 if (removeJavacState && javacState.exists()) { 144 javacState.delete(); 145 } 146 newJavacState = false; 147 if (!javacState.exists()) { 148 newJavacState = true; 149 // If there is no javac_state then delete the contents of all the artifact dirs! 150 // We do not want to risk building a broken incremental build. 151 // BUT since the makefiles still copy things straight into the bin_dir et al, 152 // we avoid deleting files here, if the option --permit-unidentified-classes was supplied. 153 if (!options.areUnidentifiedArtifactsPermitted()) { 154 deleteContents(binDir); 155 deleteContents(gensrcDir); 156 deleteContents(headerDir); 157 } 158 needsSaving = true; 159 } 160 prev = new BuildState(); 161 now = new BuildState(); 162 taintedPackages = new HashSet<>(); 163 recompiledPackages = new HashSet<>(); 164 packagesWithChangedPublicApis = new HashSet<>(); 165 } 166 167 public BuildState prev() { return prev; } 168 public BuildState now() { return now; } 169 170 /** 171 * Remove args not affecting the state. 172 */ 173 static String[] removeArgsNotAffectingState(String[] args) { 174 String[] out = new String[args.length]; 175 int j = 0; 176 for (int i = 0; i<args.length; ++i) { 177 if (args[i].equals("-j")) { 178 // Just skip it and skip following value 179 i++; 180 } else if (args[i].startsWith("--server:")) { 181 // Just skip it. 182 } else if (args[i].startsWith("--log=")) { 183 // Just skip it. 184 } else if (args[i].equals("--compare-found-sources")) { 185 // Just skip it and skip verify file name 186 i++; 187 } else { 188 // Copy argument. 189 out[j] = args[i]; 190 j++; 191 } 192 } 193 String[] ret = new String[j]; 194 System.arraycopy(out, 0, ret, 0, j); 195 return ret; 196 } 197 198 /** 199 * Specify which sources are visible to the compiler through -sourcepath. 200 */ 201 public void setVisibleSources(Map<String,Source> vs) { 202 visibleSrcs = new HashSet<>(); 203 for (String s : vs.keySet()) { 204 Source src = vs.get(s); 205 visibleSrcs.add(src.file().toURI()); 206 } 207 } 208 209 /** 210 * Returns true if this is an incremental build. 211 */ 212 public boolean isIncremental() { 213 return !prev.sources().isEmpty(); 214 } 215 216 /** 217 * Find all artifacts that exists on disk. 218 */ 219 public void findAllArtifacts() { 220 binArtifacts = findAllFiles(binDir); 221 gensrcArtifacts = findAllFiles(gensrcDir); 222 headerArtifacts = findAllFiles(headerDir); 223 } 224 225 /** 226 * Lookup the artifacts generated for this package in the previous build. 227 */ 228 private Map<String,File> fetchPrevArtifacts(String pkg) { 229 Package p = prev.packages().get(pkg); 230 if (p != null) { 231 return p.artifacts(); 232 } 233 return new HashMap<>(); 234 } 235 236 /** 237 * Delete all prev artifacts in the currently tainted packages. 238 */ 239 public void deleteClassArtifactsInTaintedPackages() { 240 for (String pkg : taintedPackages) { 241 Map<String,File> arts = fetchPrevArtifacts(pkg); 242 for (File f : arts.values()) { 243 if (f.exists() && f.getName().endsWith(".class")) { 244 f.delete(); 245 } 246 } 247 } 248 } 249 250 /** 251 * Mark the javac_state file to be in need of saving and as a side effect, 252 * it gets a new timestamp. 253 */ 254 private void needsSaving() { 255 needsSaving = true; 256 } 257 258 /** 259 * Save the javac_state file. 260 */ 261 public void save() throws IOException { 262 if (!needsSaving) 263 return; 264 try (FileWriter out = new FileWriter(javacState)) { 265 StringBuilder b = new StringBuilder(); 266 long millisNow = System.currentTimeMillis(); 267 Date d = new Date(millisNow); 268 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS"); 269 b.append("# javac_state ver 0.4 generated "+millisNow+" "+df.format(d)+"\n"); 270 b.append("# This format might change at any time. Please do not depend on it.\n"); 271 b.append("# R arguments\n"); 272 b.append("# M module\n"); 273 b.append("# P package\n"); 274 b.append("# S C source_tobe_compiled timestamp\n"); 275 b.append("# S L link_only_source timestamp\n"); 276 b.append("# G C generated_source timestamp\n"); 277 b.append("# A artifact timestamp\n"); 278 b.append("# D S dependant -> source dependency\n"); 279 b.append("# D C dependant -> classpath dependency\n"); 280 b.append("# I pubapi\n"); 281 b.append("R ").append(theArgs).append("\n"); 282 283 // Copy over the javac_state for the packages that did not need recompilation. 284 now.copyPackagesExcept(prev, recompiledPackages, new HashSet<String>()); 285 // Save the packages, ie package names, dependencies, pubapis and artifacts! 286 // I.e. the lot. 287 Module.saveModules(now.modules(), b); 288 289 String s = b.toString(); 290 out.write(s, 0, s.length()); 291 } 292 } 293 294 /** 295 * Load a javac_state file. 296 */ 297 public static JavacState load(Options options, Writer out, Writer err) { 298 JavacState db = new JavacState(options, false, out, err); 299 Module lastModule = null; 300 Package lastPackage = null; 301 Source lastSource = null; 302 boolean noFileFound = false; 303 boolean foundCorrectVerNr = false; 304 boolean newCommandLine = false; 305 boolean syntaxError = false; 306 307 Log.debug("Loading javac state file: " + db.javacState); 308 309 try (BufferedReader in = new BufferedReader(new FileReader(db.javacState))) { 310 for (;;) { 311 String l = in.readLine(); 312 if (l==null) break; 313 if (l.length()>=3 && l.charAt(1) == ' ') { 314 char c = l.charAt(0); 315 if (c == 'M') { 316 lastModule = db.prev.loadModule(l); 317 } else 318 if (c == 'P') { 319 if (lastModule == null) { syntaxError = true; break; } 320 lastPackage = db.prev.loadPackage(lastModule, l); 321 } else 322 if (c == 'D') { 323 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 324 char depType = l.charAt(2); 325 if (depType != 'S' && depType != 'C') 326 throw new RuntimeException("Bad dependency string: " + l); 327 lastPackage.parseAndAddDependency(l.substring(4), depType == 'C'); 328 } else 329 if (c == 'I') { 330 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 331 lastPackage.getPubApi().appendItem(l.substring(2)); // Strip "I " 332 } else 333 if (c == 'A') { 334 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 335 lastPackage.loadArtifact(l); 336 } else 337 if (c == 'S') { 338 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 339 lastSource = db.prev.loadSource(lastPackage, l, false); 340 } else 341 if (c == 'G') { 342 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 343 lastSource = db.prev.loadSource(lastPackage, l, true); 344 } else 345 if (c == 'R') { 346 String ncmdl = "R "+db.theArgs; 347 if (!l.equals(ncmdl)) { 348 newCommandLine = true; 349 } 350 } else 351 if (c == '#') { 352 if (l.startsWith("# javac_state ver ")) { 353 int sp = l.indexOf(" ", 18); 354 if (sp != -1) { 355 String ver = l.substring(18,sp); 356 if (!ver.equals("0.4")) { 357 break; 358 } 359 foundCorrectVerNr = true; 360 } 361 } 362 } 363 } 364 } 365 } catch (FileNotFoundException | NoSuchFileException e) { 366 // Silently create a new javac_state file. 367 noFileFound = true; 368 } catch (IOException e) { 369 Log.info("Dropping old javac_state because of errors when reading it."); 370 db = new JavacState(options, true, out, err); 371 foundCorrectVerNr = true; 372 newCommandLine = false; 373 syntaxError = false; 374 } 375 if (foundCorrectVerNr == false && !noFileFound) { 376 Log.info("Dropping old javac_state since it is of an old version."); 377 db = new JavacState(options, true, out, err); 378 } else 379 if (newCommandLine == true && !noFileFound) { 380 Log.info("Dropping old javac_state since a new command line is used!"); 381 db = new JavacState(options, true, out, err); 382 } else 383 if (syntaxError == true) { 384 Log.info("Dropping old javac_state since it contains syntax errors."); 385 db = new JavacState(options, true, out, err); 386 } 387 db.prev.calculateDependents(); 388 return db; 389 } 390 391 /** 392 * Mark a java package as tainted, ie it needs recompilation. 393 */ 394 public void taintPackage(String name, String because) { 395 if (!taintedPackages.contains(name)) { 396 if (because != null) Log.debug("Tainting "+Util.justPackageName(name)+" because "+because); 397 // It has not been tainted before. 398 taintedPackages.add(name); 399 needsSaving(); 400 Package nowp = now.packages().get(name); 401 if (nowp != null) { 402 for (String d : nowp.dependents()) { 403 taintPackage(d, because); 404 } 405 } 406 } 407 } 408 409 /** 410 * This packages need recompilation. 411 */ 412 public Set<String> taintedPackages() { 413 return taintedPackages; 414 } 415 416 /** 417 * Clean out the tainted package set, used after the first round of compiles, 418 * prior to propagating dependencies. 419 */ 420 public void clearTaintedPackages() { 421 taintedPackages = new HashSet<>(); 422 } 423 424 /** 425 * Go through all sources and check which have been removed, added or modified 426 * and taint the corresponding packages. 427 */ 428 public void checkSourceStatus(boolean check_gensrc) { 429 removedSources = calculateRemovedSources(); 430 for (Source s : removedSources) { 431 if (!s.isGenerated() || check_gensrc) { 432 taintPackage(s.pkg().name(), "source "+s.name()+" was removed"); 433 } 434 } 435 436 addedSources = calculateAddedSources(); 437 for (Source s : addedSources) { 438 String msg = null; 439 if (isIncremental()) { 440 // When building from scratch, there is no point 441 // printing "was added" for every file since all files are added. 442 // However for an incremental build it makes sense. 443 msg = "source "+s.name()+" was added"; 444 } 445 if (!s.isGenerated() || check_gensrc) { 446 taintPackage(s.pkg().name(), msg); 447 } 448 } 449 450 modifiedSources = calculateModifiedSources(); 451 for (Source s : modifiedSources) { 452 if (!s.isGenerated() || check_gensrc) { 453 taintPackage(s.pkg().name(), "source "+s.name()+" was modified"); 454 } 455 } 456 } 457 458 /** 459 * Acquire the compile_java_packages suffix rule for .java files. 460 */ 461 public Map<String,Transformer> getJavaSuffixRule() { 462 Map<String,Transformer> sr = new HashMap<>(); 463 sr.put(".java", compileJavaPackages); 464 return sr; 465 } 466 467 468 /** 469 * If artifacts have gone missing, force a recompile of the packages 470 * they belong to. 471 */ 472 public void taintPackagesThatMissArtifacts() { 473 for (Package pkg : prev.packages().values()) { 474 for (File f : pkg.artifacts().values()) { 475 if (!f.exists()) { 476 // Hmm, the artifact on disk does not exist! Someone has removed it.... 477 // Lets rebuild the package. 478 taintPackage(pkg.name(), ""+f+" is missing."); 479 } 480 } 481 } 482 } 483 484 /** 485 * Propagate recompilation through the dependency chains. 486 * Avoid re-tainting packages that have already been compiled. 487 */ 488 public void taintPackagesDependingOnChangedPackages(Set<String> pkgsWithChangedPubApi, Set<String> recentlyCompiled) { 489 // For each to-be-recompiled-candidates... 490 for (Package pkg : new HashSet<>(prev.packages().values())) { 491 // Find out what it depends upon... 492 Set<String> deps = pkg.typeDependencies() 493 .values() 494 .stream() 495 .flatMap(s -> s.stream()) 496 .collect(Collectors.toSet()); 497 for (String dep : deps) { 498 String depPkg = ":" + dep.substring(0, dep.lastIndexOf('.')); 499 if (depPkg.equals(pkg.name())) 500 continue; 501 // Checking if that dependency has changed 502 if (pkgsWithChangedPubApi.contains(depPkg) && !recentlyCompiled.contains(pkg.name())) { 503 taintPackage(pkg.name(), "its depending on " + depPkg); 504 } 505 } 506 } 507 } 508 509 /** 510 * Compare the javac_state recorded public apis of packages on the classpath 511 * with the actual public apis on the classpath. 512 */ 513 public void taintPackagesDependingOnChangedClasspathPackages() throws IOException { 514 515 // 1. Collect fully qualified names of all interesting classpath dependencies 516 Set<String> fqDependencies = new HashSet<>(); 517 for (Package pkg : prev.packages().values()) { 518 // Check if this package was compiled. If it's presence is recorded 519 // because it was on the class path and we needed to save it's 520 // public api, it's not a candidate for tainting. 521 if (pkg.sources().isEmpty()) 522 continue; 523 524 pkg.typeClasspathDependencies().values().forEach(fqDependencies::addAll); 525 } 526 527 // 2. Extract the public APIs from the on disk .class files 528 // (Reason for doing step 1 in a separate phase is to avoid extracting 529 // public APIs of the same class twice.) 530 PubApiExtractor pubApiExtractor = new PubApiExtractor(options); 531 Map<String, PubApi> onDiskPubApi = new HashMap<>(); 532 for (String cpDep : fqDependencies) { 533 onDiskPubApi.put(cpDep, pubApiExtractor.getPubApi(cpDep)); 534 } 535 pubApiExtractor.close(); 536 537 // 3. Compare them with the public APIs as of last compilation (loaded from javac_state) 538 nextPkg: 539 for (Package pkg : prev.packages().values()) { 540 // Check if this package was compiled. If it's presence is recorded 541 // because it was on the class path and we needed to save it's 542 // public api, it's not a candidate for tainting. 543 if (pkg.sources().isEmpty()) 544 continue; 545 546 Set<String> cpDepsOfThisPkg = new HashSet<>(); 547 for (Set<String> cpDeps : pkg.typeClasspathDependencies().values()) 548 cpDepsOfThisPkg.addAll(cpDeps); 549 550 for (String fqDep : cpDepsOfThisPkg) { 551 552 String depPkg = ":" + fqDep.substring(0, fqDep.lastIndexOf('.')); 553 PubApi prevPkgApi = prev.packages().get(depPkg).getPubApi(); 554 555 // This PubApi directly lists the members of the class, 556 // i.e. [ MEMBER1, MEMBER2, ... ] 557 PubApi prevDepApi = prevPkgApi.types.get(fqDep).pubApi; 558 559 // In order to dive *into* the class, we need to add 560 // .types.get(fqDep).pubApi below. 561 PubApi currentDepApi = onDiskPubApi.get(fqDep).types.get(fqDep).pubApi; 562 563 if (!currentDepApi.isBackwardCompatibleWith(prevDepApi)) { 564 List<String> apiDiff = currentDepApi.diff(prevDepApi); 565 taintPackage(pkg.name(), "depends on classpath " 566 + "package which has an updated package api: " 567 + String.join("\n", apiDiff)); 568 //Log.debug("========================================"); 569 //Log.debug("------ PREV API ------------------------"); 570 //prevDepApi.asListOfStrings().forEach(Log::debug); 571 //Log.debug("------ CURRENT API ---------------------"); 572 //currentDepApi.asListOfStrings().forEach(Log::debug); 573 //Log.debug("========================================"); 574 continue nextPkg; 575 } 576 } 577 } 578 } 579 580 /** 581 * Scan all output dirs for artifacts and remove those files (artifacts?) 582 * that are not recognized as such, in the javac_state file. 583 */ 584 public void removeUnidentifiedArtifacts() { 585 Set<File> allKnownArtifacts = new HashSet<>(); 586 for (Package pkg : prev.packages().values()) { 587 for (File f : pkg.artifacts().values()) { 588 allKnownArtifacts.add(f); 589 } 590 } 591 // Do not forget about javac_state.... 592 allKnownArtifacts.add(javacState); 593 594 for (File f : binArtifacts) { 595 if (!allKnownArtifacts.contains(f) && 596 !options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) { 597 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 598 f.delete(); 599 } 600 } 601 for (File f : headerArtifacts) { 602 if (!allKnownArtifacts.contains(f)) { 603 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 604 f.delete(); 605 } 606 } 607 for (File f : gensrcArtifacts) { 608 if (!allKnownArtifacts.contains(f)) { 609 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 610 f.delete(); 611 } 612 } 613 } 614 615 /** 616 * Remove artifacts that are no longer produced when compiling! 617 */ 618 public void removeSuperfluousArtifacts(Set<String> recentlyCompiled) { 619 // Nothing to do, if nothing was recompiled. 620 if (recentlyCompiled.size() == 0) return; 621 622 for (String pkg : now.packages().keySet()) { 623 // If this package has not been recompiled, skip the check. 624 if (!recentlyCompiled.contains(pkg)) continue; 625 Collection<File> arts = now.artifacts().values(); 626 for (File f : fetchPrevArtifacts(pkg).values()) { 627 if (!arts.contains(f)) { 628 Log.debug("Removing "+f.getPath()+" since it is now superfluous!"); 629 if (f.exists()) f.delete(); 630 } 631 } 632 } 633 } 634 635 /** 636 * Return those files belonging to prev, but not now. 637 */ 638 private Set<Source> calculateRemovedSources() { 639 Set<Source> removed = new HashSet<>(); 640 for (String src : prev.sources().keySet()) { 641 if (now.sources().get(src) == null) { 642 removed.add(prev.sources().get(src)); 643 } 644 } 645 return removed; 646 } 647 648 /** 649 * Return those files belonging to now, but not prev. 650 */ 651 private Set<Source> calculateAddedSources() { 652 Set<Source> added = new HashSet<>(); 653 for (String src : now.sources().keySet()) { 654 if (prev.sources().get(src) == null) { 655 added.add(now.sources().get(src)); 656 } 657 } 658 return added; 659 } 660 661 /** 662 * Return those files where the timestamp is newer. 663 * If a source file timestamp suddenly is older than what is known 664 * about it in javac_state, then consider it modified, but print 665 * a warning! 666 */ 667 private Set<Source> calculateModifiedSources() { 668 Set<Source> modified = new HashSet<>(); 669 for (String src : now.sources().keySet()) { 670 Source n = now.sources().get(src); 671 Source t = prev.sources().get(src); 672 if (prev.sources().get(src) != null) { 673 if (t != null) { 674 if (n.lastModified() > t.lastModified()) { 675 modified.add(n); 676 } else if (n.lastModified() < t.lastModified()) { 677 modified.add(n); 678 Log.warn("The source file "+n.name()+" timestamp has moved backwards in time."); 679 } 680 } 681 } 682 } 683 return modified; 684 } 685 686 /** 687 * Recursively delete a directory and all its contents. 688 */ 689 private void deleteContents(File dir) { 690 if (dir != null && dir.exists()) { 691 for (File f : dir.listFiles()) { 692 if (f.isDirectory()) { 693 deleteContents(f); 694 } 695 if (!options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) { 696 Log.debug("Removing "+f.getAbsolutePath()); 697 f.delete(); 698 } 699 } 700 } 701 } 702 703 /** 704 * Run the copy translator only. 705 */ 706 public void performCopying(File binDir, Map<String,Transformer> suffixRules) { 707 Map<String,Transformer> sr = new HashMap<>(); 708 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) { 709 if (e.getValue().getClass().equals(CopyFile.class)) { 710 sr.put(e.getKey(), e.getValue()); 711 } 712 } 713 perform(null, binDir, sr); 714 } 715 716 /** 717 * Run all the translators that translate into java source code. 718 * I.e. all translators that are not copy nor compile_java_source. 719 */ 720 public void performTranslation(File gensrcDir, Map<String,Transformer> suffixRules) { 721 Map<String,Transformer> sr = new HashMap<>(); 722 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) { 723 Class<?> trClass = e.getValue().getClass(); 724 if (trClass == CompileJavaPackages.class || trClass == CopyFile.class) 725 continue; 726 727 sr.put(e.getKey(), e.getValue()); 728 } 729 perform(null, gensrcDir, sr); 730 } 731 732 /** 733 * Compile all the java sources. Return true, if it needs to be called again! 734 */ 735 public boolean performJavaCompilations(CompilationService sjavac, 736 Options args, 737 Set<String> recentlyCompiled, 738 boolean[] rcValue) { 739 Map<String,Transformer> suffixRules = new HashMap<>(); 740 suffixRules.put(".java", compileJavaPackages); 741 compileJavaPackages.setExtra(args); 742 rcValue[0] = perform(sjavac, binDir, suffixRules); 743 recentlyCompiled.addAll(taintedPackages()); 744 clearTaintedPackages(); 745 boolean again = !packagesWithChangedPublicApis.isEmpty(); 746 taintPackagesDependingOnChangedPackages(packagesWithChangedPublicApis, recentlyCompiled); 747 packagesWithChangedPublicApis = new HashSet<>(); 748 return again && rcValue[0]; 749 750 // TODO: Figure out why 'again' checks packagesWithChangedPublicAPis. 751 // (It shouldn't matter if packages had changed pub apis as long as no 752 // one depends on them. Wouldn't it make more sense to let 'again' 753 // depend on taintedPackages?) 754 } 755 756 /** 757 * Store the source into the set of sources belonging to the given transform. 758 */ 759 private void addFileToTransform(Map<Transformer,Map<String,Set<URI>>> gs, Transformer t, Source s) { 760 Map<String,Set<URI>> fs = gs.get(t); 761 if (fs == null) { 762 fs = new HashMap<>(); 763 gs.put(t, fs); 764 } 765 Set<URI> ss = fs.get(s.pkg().name()); 766 if (ss == null) { 767 ss = new HashSet<>(); 768 fs.put(s.pkg().name(), ss); 769 } 770 ss.add(s.file().toURI()); 771 } 772 773 /** 774 * For all packages, find all sources belonging to the package, group the sources 775 * based on their transformers and apply the transformers on each source code group. 776 */ 777 private boolean perform(CompilationService sjavac, 778 File outputDir, 779 Map<String,Transformer> suffixRules) { 780 boolean rc = true; 781 // Group sources based on transforms. A source file can only belong to a single transform. 782 Map<Transformer,Map<String,Set<URI>>> groupedSources = new HashMap<>(); 783 for (Source src : now.sources().values()) { 784 Transformer t = suffixRules.get(src.suffix()); 785 if (t != null) { 786 if (taintedPackages.contains(src.pkg().name()) && !src.isLinkedOnly()) { 787 addFileToTransform(groupedSources, t, src); 788 } 789 } 790 } 791 // Go through the transforms and transform them. 792 for (Map.Entry<Transformer, Map<String, Set<URI>>> e : groupedSources.entrySet()) { 793 Transformer t = e.getKey(); 794 Map<String, Set<URI>> srcs = e.getValue(); 795 // These maps need to be synchronized since multiple threads will be 796 // writing results into them. 797 Map<String, Set<URI>> packageArtifacts = Collections.synchronizedMap(new HashMap<>()); 798 Map<String, Map<String, Set<String>>> packageDependencies = Collections.synchronizedMap(new HashMap<>()); 799 Map<String, Map<String, Set<String>>> packageCpDependencies = Collections.synchronizedMap(new HashMap<>()); 800 Map<String, PubApi> packagePublicApis = Collections.synchronizedMap(new HashMap<>()); 801 Map<String, PubApi> dependencyPublicApis = Collections.synchronizedMap(new HashMap<>()); 802 803 boolean r = t.transform(sjavac, 804 srcs, 805 visibleSrcs, 806 prev.dependents(), 807 outputDir.toURI(), 808 packageArtifacts, 809 packageDependencies, 810 packageCpDependencies, 811 packagePublicApis, 812 dependencyPublicApis, 813 0, 814 isIncremental(), 815 numCores, 816 out, 817 err); 818 if (!r) 819 rc = false; 820 821 for (String p : srcs.keySet()) { 822 recompiledPackages.add(p); 823 } 824 // The transform is done! Extract all the artifacts and store the info into the Package objects. 825 for (Map.Entry<String, Set<URI>> a : packageArtifacts.entrySet()) { 826 Module mnow = now.findModuleFromPackageName(a.getKey()); 827 mnow.addArtifacts(a.getKey(), a.getValue()); 828 } 829 // Extract all the dependencies and store the info into the Package objects. 830 for (Map.Entry<String, Map<String, Set<String>>> a : packageDependencies.entrySet()) { 831 Map<String, Set<String>> deps = a.getValue(); 832 Module mnow = now.findModuleFromPackageName(a.getKey()); 833 mnow.setDependencies(a.getKey(), deps, false); 834 } 835 for (Map.Entry<String, Map<String, Set<String>>> a : packageCpDependencies.entrySet()) { 836 Map<String, Set<String>> deps = a.getValue(); 837 Module mnow = now.findModuleFromPackageName(a.getKey()); 838 mnow.setDependencies(a.getKey(), deps, true); 839 } 840 841 // This map contains the public api of the types that this 842 // compilation depended upon. This means that it may not contain 843 // full packages. In other words, we shouldn't remove knowledge of 844 // public apis but merge these with what we already have. 845 for (Map.Entry<String, PubApi> a : dependencyPublicApis.entrySet()) { 846 String pkg = a.getKey(); 847 PubApi packagePartialPubApi = a.getValue(); 848 Package pkgNow = now.findModuleFromPackageName(pkg).lookupPackage(pkg); 849 PubApi currentPubApi = pkgNow.getPubApi(); 850 PubApi newPubApi = PubApi.mergeTypes(currentPubApi, packagePartialPubApi); 851 pkgNow.setPubapi(newPubApi); 852 853 // See JDK-8071904 854 if (now.packages().containsKey(pkg)) 855 now.packages().get(pkg).setPubapi(newPubApi); 856 else 857 now.packages().put(pkg, pkgNow); 858 } 859 860 // The packagePublicApis cover entire packages (since sjavac compiles 861 // stuff on package level). This means that if a type is missing 862 // in the public api of a given package, it means that it has been 863 // removed. In other words, we should *set* the pubapi to whatever 864 // this map contains, and not merge it with what we already have. 865 for (Map.Entry<String, PubApi> a : packagePublicApis.entrySet()) { 866 String pkg = a.getKey(); 867 PubApi newPubApi = a.getValue(); 868 Module mprev = prev.findModuleFromPackageName(pkg); 869 Module mnow = now.findModuleFromPackageName(pkg); 870 mnow.setPubapi(pkg, newPubApi); 871 if (mprev.hasPubapiChanged(pkg, newPubApi)) { 872 // Aha! The pubapi of this package has changed! 873 // It can also be a new compile from scratch. 874 if (mprev.lookupPackage(pkg).existsInJavacState()) { 875 // This is an incremental compile! The pubapi 876 // did change. Trigger recompilation of dependents. 877 packagesWithChangedPublicApis.add(pkg); 878 Log.info("The API of " + Util.justPackageName(pkg) + " has changed!"); 879 } 880 } 881 } 882 } 883 return rc; 884 } 885 886 /** 887 * Utility method to recursively find all files below a directory. 888 */ 889 private static Set<File> findAllFiles(File dir) { 890 Set<File> foundFiles = new HashSet<>(); 891 if (dir == null) { 892 return foundFiles; 893 } 894 recurse(dir, foundFiles); 895 return foundFiles; 896 } 897 898 private static void recurse(File dir, Set<File> foundFiles) { 899 for (File f : dir.listFiles()) { 900 if (f.isFile()) { 901 foundFiles.add(f); 902 } else if (f.isDirectory()) { 903 recurse(f, foundFiles); 904 } 905 } 906 } 907 908 /** 909 * Compare the calculate source list, with an explicit list, usually 910 * supplied from the makefile. Used to detect bugs where the makefile and 911 * sjavac have different opinions on which files should be compiled. 912 */ 913 public void compareWithMakefileList(File makefileSourceList) 914 throws ProblemException { 915 // If we are building on win32 using for example cygwin the paths in the 916 // makefile source list 917 // might be /cygdrive/c/.... which does not match c:\.... 918 // We need to adjust our calculated sources to be identical, if 919 // necessary. 920 boolean mightNeedRewriting = File.pathSeparatorChar == ';'; 921 922 if (makefileSourceList == null) 923 return; 924 925 Set<String> calculatedSources = new HashSet<>(); 926 Set<String> listedSources = new HashSet<>(); 927 928 // Create a set of filenames with full paths. 929 for (Source s : now.sources().values()) { 930 // Don't include link only sources when comparing sources to compile 931 if (!s.isLinkedOnly()) { 932 String path = s.file().getPath(); 933 if (mightNeedRewriting) 934 path = Util.normalizeDriveLetter(path); 935 calculatedSources.add(path); 936 } 937 } 938 // Read in the file and create another set of filenames with full paths. 939 try(BufferedReader in = new BufferedReader(new FileReader(makefileSourceList))) { 940 for (;;) { 941 String l = in.readLine(); 942 if (l==null) break; 943 l = l.trim(); 944 if (mightNeedRewriting) { 945 if (l.indexOf(":") == 1 && l.indexOf("\\") == 2) { 946 // Everything a-ok, the format is already C:\foo\bar 947 } else if (l.indexOf(":") == 1 && l.indexOf("/") == 2) { 948 // The format is C:/foo/bar, rewrite into the above format. 949 l = l.replaceAll("/","\\\\"); 950 } else if (l.charAt(0) == '/' && l.indexOf("/",1) != -1) { 951 // The format might be: /cygdrive/c/foo/bar, rewrite into the above format. 952 // Do not hardcode the name cygdrive here. 953 int slash = l.indexOf("/",1); 954 l = l.replaceAll("/","\\\\"); 955 l = ""+l.charAt(slash+1)+":"+l.substring(slash+2); 956 } 957 if (Character.isLowerCase(l.charAt(0))) { 958 l = Character.toUpperCase(l.charAt(0))+l.substring(1); 959 } 960 } 961 listedSources.add(l); 962 } 963 } catch (FileNotFoundException | NoSuchFileException e) { 964 throw new ProblemException("Could not open "+makefileSourceList.getPath()+" since it does not exist!"); 965 } catch (IOException e) { 966 throw new ProblemException("Could not read "+makefileSourceList.getPath()); 967 } 968 969 for (String s : listedSources) { 970 if (!calculatedSources.contains(s)) { 971 throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!"); 972 } 973 } 974 975 for (String s : calculatedSources) { 976 if (!listedSources.contains(s)) { 977 throw new ProblemException("The smart javac wrapper calculated source "+s+" was not listed by the makefiles!"); 978 } 979 } 980 } 981} 982