JavacState.java revision 3012:adba44f6b471
1/* 2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26package com.sun.tools.sjavac; 27 28import java.io.BufferedReader; 29import java.io.File; 30import java.io.FileNotFoundException; 31import java.io.FileReader; 32import java.io.FileWriter; 33import java.io.IOException; 34import java.io.PrintStream; 35import java.net.URI; 36import java.nio.file.NoSuchFileException; 37import java.text.SimpleDateFormat; 38import java.util.Collection; 39import java.util.Collections; 40import java.util.Date; 41import java.util.HashMap; 42import java.util.HashSet; 43import java.util.List; 44import java.util.Map; 45import java.util.Set; 46import java.util.stream.Collectors; 47 48import com.sun.tools.sjavac.comp.CompilationService; 49import com.sun.tools.sjavac.options.Options; 50import com.sun.tools.sjavac.pubapi.PubApi; 51 52/** 53 * The javac state class maintains the previous (prev) and the current (now) 54 * build states and everything else that goes into the javac_state file. 55 * 56 * <p><b>This is NOT part of any supported API. 57 * If you write code that depends on this, you do so at your own risk. 58 * This code and its internal interfaces are subject to change or 59 * deletion without notice.</b> 60 */ 61public class JavacState { 62 // The arguments to the compile. If not identical, then it cannot 63 // be an incremental build! 64 String theArgs; 65 // The number of cores limits how many threads are used for heavy concurrent work. 66 int numCores; 67 68 // The bin_dir/javac_state 69 private File javacState; 70 71 // The previous build state is loaded from javac_state 72 private BuildState prev; 73 // The current build state is constructed during the build, 74 // then saved as the new javac_state. 75 private BuildState now; 76 77 // Something has changed in the javac_state. It needs to be saved! 78 private boolean needsSaving; 79 // If this is a new javac_state file, then do not print unnecessary messages. 80 private boolean newJavacState; 81 82 // These are packages where something has changed and the package 83 // needs to be recompiled. Actions that trigger recompilation: 84 // * source belonging to the package has changed 85 // * artifact belonging to the package is lost, or its timestamp has been changed. 86 // * an unknown artifact has appeared, we simply delete it, but we also trigger a recompilation. 87 // * a package that is tainted, taints all packages that depend on it. 88 private Set<String> taintedPackages; 89 // After a compile, the pubapis are compared with the pubapis stored in the javac state file. 90 // Any packages where the pubapi differ are added to this set. 91 // Later we use this set and the dependency information to taint dependent packages. 92 private Set<String> packagesWithChangedPublicApis; 93 // When a module-info.java file is changed, taint the module, 94 // then taint all modules that depend on that that module. 95 // A module dependency can occur directly through a require, or 96 // indirectly through a module that does a public export for the first tainted module. 97 // When all modules are tainted, then taint all packages belonging to these modules. 98 // Then rebuild. It is perhaps possible (and valuable?) to do a more finegrained examination of the 99 // change in module-info.java, but that will have to wait. 100 private Set<String> taintedModules; 101 // The set of all packages that has been recompiled. 102 // Copy over the javac_state for the packages that did not need recompilation, 103 // verbatim from the previous (prev) to the new (now) build state. 104 private Set<String> recompiledPackages; 105 106 // The output directories filled with tasty artifacts. 107 private File binDir, gensrcDir, headerDir, stateDir; 108 109 // The current status of the file system. 110 private Set<File> binArtifacts; 111 private Set<File> gensrcArtifacts; 112 private Set<File> headerArtifacts; 113 114 // The status of the sources. 115 Set<Source> removedSources = null; 116 Set<Source> addedSources = null; 117 Set<Source> modifiedSources = null; 118 119 // Visible sources for linking. These are the only 120 // ones that -sourcepath is allowed to see. 121 Set<URI> visibleSrcs; 122 123 // Visible classes for linking. These are the only 124 // ones that -classpath is allowed to see. 125 // It maps from a classpath root to the set of visible classes for that root. 126 // If the set is empty, then all classes are visible for that root. 127 // It can also map from a jar file to the set of visible classes for that jar file. 128 Map<URI,Set<String>> visibleClasses; 129 130 // Setup transform that always exist. 131 private CompileJavaPackages compileJavaPackages = new CompileJavaPackages(); 132 133 // Where to send stdout and stderr. 134 private PrintStream out, err; 135 136 // Command line options. 137 private Options options; 138 139 JavacState(Options op, boolean removeJavacState, PrintStream o, PrintStream e) { 140 options = op; 141 out = o; 142 err = e; 143 numCores = options.getNumCores(); 144 theArgs = options.getStateArgsString(); 145 binDir = Util.pathToFile(options.getDestDir()); 146 gensrcDir = Util.pathToFile(options.getGenSrcDir()); 147 headerDir = Util.pathToFile(options.getHeaderDir()); 148 stateDir = Util.pathToFile(options.getStateDir()); 149 javacState = new File(stateDir, "javac_state"); 150 if (removeJavacState && javacState.exists()) { 151 javacState.delete(); 152 } 153 newJavacState = false; 154 if (!javacState.exists()) { 155 newJavacState = true; 156 // If there is no javac_state then delete the contents of all the artifact dirs! 157 // We do not want to risk building a broken incremental build. 158 // BUT since the makefiles still copy things straight into the bin_dir et al, 159 // we avoid deleting files here, if the option --permit-unidentified-classes was supplied. 160 if (!options.areUnidentifiedArtifactsPermitted()) { 161 deleteContents(binDir); 162 deleteContents(gensrcDir); 163 deleteContents(headerDir); 164 } 165 needsSaving = true; 166 } 167 prev = new BuildState(); 168 now = new BuildState(); 169 taintedPackages = new HashSet<>(); 170 recompiledPackages = new HashSet<>(); 171 packagesWithChangedPublicApis = new HashSet<>(); 172 } 173 174 public BuildState prev() { return prev; } 175 public BuildState now() { return now; } 176 177 /** 178 * Remove args not affecting the state. 179 */ 180 static String[] removeArgsNotAffectingState(String[] args) { 181 String[] out = new String[args.length]; 182 int j = 0; 183 for (int i = 0; i<args.length; ++i) { 184 if (args[i].equals("-j")) { 185 // Just skip it and skip following value 186 i++; 187 } else if (args[i].startsWith("--server:")) { 188 // Just skip it. 189 } else if (args[i].startsWith("--log=")) { 190 // Just skip it. 191 } else if (args[i].equals("--compare-found-sources")) { 192 // Just skip it and skip verify file name 193 i++; 194 } else { 195 // Copy argument. 196 out[j] = args[i]; 197 j++; 198 } 199 } 200 String[] ret = new String[j]; 201 System.arraycopy(out, 0, ret, 0, j); 202 return ret; 203 } 204 205 /** 206 * Specify which sources are visible to the compiler through -sourcepath. 207 */ 208 public void setVisibleSources(Map<String,Source> vs) { 209 visibleSrcs = new HashSet<>(); 210 for (String s : vs.keySet()) { 211 Source src = vs.get(s); 212 visibleSrcs.add(src.file().toURI()); 213 } 214 } 215 216 /** 217 * Specify which classes are visible to the compiler through -classpath. 218 */ 219 public void setVisibleClasses(Map<String,Source> vs) { 220 visibleSrcs = new HashSet<>(); 221 for (String s : vs.keySet()) { 222 Source src = vs.get(s); 223 visibleSrcs.add(src.file().toURI()); 224 } 225 } 226 /** 227 * Returns true if this is an incremental build. 228 */ 229 public boolean isIncremental() { 230 return !prev.sources().isEmpty(); 231 } 232 233 /** 234 * Find all artifacts that exists on disk. 235 */ 236 public void findAllArtifacts() { 237 binArtifacts = findAllFiles(binDir); 238 gensrcArtifacts = findAllFiles(gensrcDir); 239 headerArtifacts = findAllFiles(headerDir); 240 } 241 242 /** 243 * Lookup the artifacts generated for this package in the previous build. 244 */ 245 private Map<String,File> fetchPrevArtifacts(String pkg) { 246 Package p = prev.packages().get(pkg); 247 if (p != null) { 248 return p.artifacts(); 249 } 250 return new HashMap<>(); 251 } 252 253 /** 254 * Delete all prev artifacts in the currently tainted packages. 255 */ 256 public void deleteClassArtifactsInTaintedPackages() { 257 for (String pkg : taintedPackages) { 258 Map<String,File> arts = fetchPrevArtifacts(pkg); 259 for (File f : arts.values()) { 260 if (f.exists() && f.getName().endsWith(".class")) { 261 f.delete(); 262 } 263 } 264 } 265 } 266 267 /** 268 * Mark the javac_state file to be in need of saving and as a side effect, 269 * it gets a new timestamp. 270 */ 271 private void needsSaving() { 272 needsSaving = true; 273 } 274 275 /** 276 * Save the javac_state file. 277 */ 278 public void save() throws IOException { 279 if (!needsSaving) 280 return; 281 try (FileWriter out = new FileWriter(javacState)) { 282 StringBuilder b = new StringBuilder(); 283 long millisNow = System.currentTimeMillis(); 284 Date d = new Date(millisNow); 285 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS"); 286 b.append("# javac_state ver 0.4 generated "+millisNow+" "+df.format(d)+"\n"); 287 b.append("# This format might change at any time. Please do not depend on it.\n"); 288 b.append("# R arguments\n"); 289 b.append("# M module\n"); 290 b.append("# P package\n"); 291 b.append("# S C source_tobe_compiled timestamp\n"); 292 b.append("# S L link_only_source timestamp\n"); 293 b.append("# G C generated_source timestamp\n"); 294 b.append("# A artifact timestamp\n"); 295 b.append("# D S dependant -> source dependency\n"); 296 b.append("# D C dependant -> classpath dependency\n"); 297 b.append("# I pubapi\n"); 298 b.append("R ").append(theArgs).append("\n"); 299 300 // Copy over the javac_state for the packages that did not need recompilation. 301 now.copyPackagesExcept(prev, recompiledPackages, new HashSet<String>()); 302 // Save the packages, ie package names, dependencies, pubapis and artifacts! 303 // I.e. the lot. 304 Module.saveModules(now.modules(), b); 305 306 String s = b.toString(); 307 out.write(s, 0, s.length()); 308 } 309 } 310 311 /** 312 * Load a javac_state file. 313 */ 314 public static JavacState load(Options options, PrintStream out, PrintStream err) { 315 JavacState db = new JavacState(options, false, out, err); 316 Module lastModule = null; 317 Package lastPackage = null; 318 Source lastSource = null; 319 boolean noFileFound = false; 320 boolean foundCorrectVerNr = false; 321 boolean newCommandLine = false; 322 boolean syntaxError = false; 323 324 Log.debug("Loading javac state file: " + db.javacState); 325 326 try (BufferedReader in = new BufferedReader(new FileReader(db.javacState))) { 327 for (;;) { 328 String l = in.readLine(); 329 if (l==null) break; 330 if (l.length()>=3 && l.charAt(1) == ' ') { 331 char c = l.charAt(0); 332 if (c == 'M') { 333 lastModule = db.prev.loadModule(l); 334 } else 335 if (c == 'P') { 336 if (lastModule == null) { syntaxError = true; break; } 337 lastPackage = db.prev.loadPackage(lastModule, l); 338 } else 339 if (c == 'D') { 340 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 341 char depType = l.charAt(2); 342 if (depType != 'S' && depType != 'C') 343 throw new RuntimeException("Bad dependency string: " + l); 344 lastPackage.parseAndAddDependency(l.substring(4), depType == 'C'); 345 } else 346 if (c == 'I') { 347 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 348 lastPackage.getPubApi().appendItem(l.substring(2)); // Strip "I " 349 } else 350 if (c == 'A') { 351 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 352 lastPackage.loadArtifact(l); 353 } else 354 if (c == 'S') { 355 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 356 lastSource = db.prev.loadSource(lastPackage, l, false); 357 } else 358 if (c == 'G') { 359 if (lastModule == null || lastPackage == null) { syntaxError = true; break; } 360 lastSource = db.prev.loadSource(lastPackage, l, true); 361 } else 362 if (c == 'R') { 363 String ncmdl = "R "+db.theArgs; 364 if (!l.equals(ncmdl)) { 365 newCommandLine = true; 366 } 367 } else 368 if (c == '#') { 369 if (l.startsWith("# javac_state ver ")) { 370 int sp = l.indexOf(" ", 18); 371 if (sp != -1) { 372 String ver = l.substring(18,sp); 373 if (!ver.equals("0.4")) { 374 break; 375 } 376 foundCorrectVerNr = true; 377 } 378 } 379 } 380 } 381 } 382 } catch (FileNotFoundException | NoSuchFileException e) { 383 // Silently create a new javac_state file. 384 noFileFound = true; 385 } catch (IOException e) { 386 Log.info("Dropping old javac_state because of errors when reading it."); 387 db = new JavacState(options, true, out, err); 388 foundCorrectVerNr = true; 389 newCommandLine = false; 390 syntaxError = false; 391 } 392 if (foundCorrectVerNr == false && !noFileFound) { 393 Log.info("Dropping old javac_state since it is of an old version."); 394 db = new JavacState(options, true, out, err); 395 } else 396 if (newCommandLine == true && !noFileFound) { 397 Log.info("Dropping old javac_state since a new command line is used!"); 398 db = new JavacState(options, true, out, err); 399 } else 400 if (syntaxError == true) { 401 Log.info("Dropping old javac_state since it contains syntax errors."); 402 db = new JavacState(options, true, out, err); 403 } 404 db.prev.calculateDependents(); 405 return db; 406 } 407 408 /** 409 * Mark a java package as tainted, ie it needs recompilation. 410 */ 411 public void taintPackage(String name, String because) { 412 if (!taintedPackages.contains(name)) { 413 if (because != null) Log.debug("Tainting "+Util.justPackageName(name)+" because "+because); 414 // It has not been tainted before. 415 taintedPackages.add(name); 416 needsSaving(); 417 Package nowp = now.packages().get(name); 418 if (nowp != null) { 419 for (String d : nowp.dependents()) { 420 taintPackage(d, because); 421 } 422 } 423 } 424 } 425 426 /** 427 * This packages need recompilation. 428 */ 429 public Set<String> taintedPackages() { 430 return taintedPackages; 431 } 432 433 /** 434 * Clean out the tainted package set, used after the first round of compiles, 435 * prior to propagating dependencies. 436 */ 437 public void clearTaintedPackages() { 438 taintedPackages = new HashSet<>(); 439 } 440 441 /** 442 * Go through all sources and check which have been removed, added or modified 443 * and taint the corresponding packages. 444 */ 445 public void checkSourceStatus(boolean check_gensrc) { 446 removedSources = calculateRemovedSources(); 447 for (Source s : removedSources) { 448 if (!s.isGenerated() || check_gensrc) { 449 taintPackage(s.pkg().name(), "source "+s.name()+" was removed"); 450 } 451 } 452 453 addedSources = calculateAddedSources(); 454 for (Source s : addedSources) { 455 String msg = null; 456 if (isIncremental()) { 457 // When building from scratch, there is no point 458 // printing "was added" for every file since all files are added. 459 // However for an incremental build it makes sense. 460 msg = "source "+s.name()+" was added"; 461 } 462 if (!s.isGenerated() || check_gensrc) { 463 taintPackage(s.pkg().name(), msg); 464 } 465 } 466 467 modifiedSources = calculateModifiedSources(); 468 for (Source s : modifiedSources) { 469 if (!s.isGenerated() || check_gensrc) { 470 taintPackage(s.pkg().name(), "source "+s.name()+" was modified"); 471 } 472 } 473 } 474 475 /** 476 * Acquire the compile_java_packages suffix rule for .java files. 477 */ 478 public Map<String,Transformer> getJavaSuffixRule() { 479 Map<String,Transformer> sr = new HashMap<>(); 480 sr.put(".java", compileJavaPackages); 481 return sr; 482 } 483 484 485 /** 486 * If artifacts have gone missing, force a recompile of the packages 487 * they belong to. 488 */ 489 public void taintPackagesThatMissArtifacts() { 490 for (Package pkg : prev.packages().values()) { 491 for (File f : pkg.artifacts().values()) { 492 if (!f.exists()) { 493 // Hmm, the artifact on disk does not exist! Someone has removed it.... 494 // Lets rebuild the package. 495 taintPackage(pkg.name(), ""+f+" is missing."); 496 } 497 } 498 } 499 } 500 501 /** 502 * Propagate recompilation through the dependency chains. 503 * Avoid re-tainting packages that have already been compiled. 504 */ 505 public void taintPackagesDependingOnChangedPackages(Set<String> pkgsWithChangedPubApi, Set<String> recentlyCompiled) { 506 // For each to-be-recompiled-candidates... 507 for (Package pkg : new HashSet<>(prev.packages().values())) { 508 // Find out what it depends upon... 509 Set<String> deps = pkg.typeDependencies() 510 .values() 511 .stream() 512 .flatMap(s -> s.stream()) 513 .collect(Collectors.toSet()); 514 for (String dep : deps) { 515 String depPkg = ":" + dep.substring(0, dep.lastIndexOf('.')); 516 if (depPkg.equals(pkg.name())) 517 continue; 518 // Checking if that dependency has changed 519 if (pkgsWithChangedPubApi.contains(depPkg) && !recentlyCompiled.contains(pkg.name())) { 520 taintPackage(pkg.name(), "its depending on " + depPkg); 521 } 522 } 523 } 524 } 525 526 /** 527 * Compare the javac_state recorded public apis of packages on the classpath 528 * with the actual public apis on the classpath. 529 */ 530 public void taintPackagesDependingOnChangedClasspathPackages() { 531 532 // 1. Collect fully qualified names of all interesting classpath dependencies 533 Set<String> fqDependencies = new HashSet<>(); 534 for (Package pkg : prev.packages().values()) { 535 // Check if this package was compiled. If it's presence is recorded 536 // because it was on the class path and we needed to save it's 537 // public api, it's not a candidate for tainting. 538 if (pkg.sources().isEmpty()) 539 continue; 540 541 pkg.typeClasspathDependencies().values().forEach(fqDependencies::addAll); 542 } 543 544 // 2. Extract the public APIs from the on disk .class files 545 // (Reason for doing step 1 in a separate phase is to avoid extracting 546 // public APIs of the same class twice.) 547 PubApiExtractor pubApiExtractor = new PubApiExtractor(options); 548 Map<String, PubApi> onDiskPubApi = new HashMap<>(); 549 for (String cpDep : fqDependencies) { 550 onDiskPubApi.put(cpDep, pubApiExtractor.getPubApi(cpDep)); 551 } 552 553 // 3. Compare them with the public APIs as of last compilation (loaded from javac_state) 554 nextPkg: 555 for (Package pkg : prev.packages().values()) { 556 // Check if this package was compiled. If it's presence is recorded 557 // because it was on the class path and we needed to save it's 558 // public api, it's not a candidate for tainting. 559 if (pkg.sources().isEmpty()) 560 continue; 561 562 Set<String> cpDepsOfThisPkg = new HashSet<>(); 563 for (Set<String> cpDeps : pkg.typeClasspathDependencies().values()) 564 cpDepsOfThisPkg.addAll(cpDeps); 565 566 for (String fqDep : cpDepsOfThisPkg) { 567 568 String depPkg = ":" + fqDep.substring(0, fqDep.lastIndexOf('.')); 569 PubApi prevPkgApi = prev.packages().get(depPkg).getPubApi(); 570 571 // This PubApi directly lists the members of the class, 572 // i.e. [ MEMBER1, MEMBER2, ... ] 573 PubApi prevDepApi = prevPkgApi.types.get(fqDep).pubApi; 574 575 // In order to dive *into* the class, we need to add 576 // .types.get(fqDep).pubApi below. 577 PubApi currentDepApi = onDiskPubApi.get(fqDep).types.get(fqDep).pubApi; 578 579 if (!currentDepApi.isBackwardCompatibleWith(prevDepApi)) { 580 List<String> apiDiff = currentDepApi.diff(prevDepApi); 581 taintPackage(pkg.name(), "depends on classpath " 582 + "package which has an updated package api: " 583 + String.join("\n", apiDiff)); 584 //Log.debug("========================================"); 585 //Log.debug("------ PREV API ------------------------"); 586 //prevDepApi.asListOfStrings().forEach(Log::debug); 587 //Log.debug("------ CURRENT API ---------------------"); 588 //currentDepApi.asListOfStrings().forEach(Log::debug); 589 //Log.debug("========================================"); 590 continue nextPkg; 591 } 592 } 593 } 594 } 595 596 /** 597 * Scan all output dirs for artifacts and remove those files (artifacts?) 598 * that are not recognized as such, in the javac_state file. 599 */ 600 public void removeUnidentifiedArtifacts() { 601 Set<File> allKnownArtifacts = new HashSet<>(); 602 for (Package pkg : prev.packages().values()) { 603 for (File f : pkg.artifacts().values()) { 604 allKnownArtifacts.add(f); 605 } 606 } 607 // Do not forget about javac_state.... 608 allKnownArtifacts.add(javacState); 609 610 for (File f : binArtifacts) { 611 if (!allKnownArtifacts.contains(f) && 612 !options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) { 613 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 614 f.delete(); 615 } 616 } 617 for (File f : headerArtifacts) { 618 if (!allKnownArtifacts.contains(f)) { 619 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 620 f.delete(); 621 } 622 } 623 for (File f : gensrcArtifacts) { 624 if (!allKnownArtifacts.contains(f)) { 625 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state."); 626 f.delete(); 627 } 628 } 629 } 630 631 /** 632 * Remove artifacts that are no longer produced when compiling! 633 */ 634 public void removeSuperfluousArtifacts(Set<String> recentlyCompiled) { 635 // Nothing to do, if nothing was recompiled. 636 if (recentlyCompiled.size() == 0) return; 637 638 for (String pkg : now.packages().keySet()) { 639 // If this package has not been recompiled, skip the check. 640 if (!recentlyCompiled.contains(pkg)) continue; 641 Collection<File> arts = now.artifacts().values(); 642 for (File f : fetchPrevArtifacts(pkg).values()) { 643 if (!arts.contains(f)) { 644 Log.debug("Removing "+f.getPath()+" since it is now superfluous!"); 645 if (f.exists()) f.delete(); 646 } 647 } 648 } 649 } 650 651 /** 652 * Return those files belonging to prev, but not now. 653 */ 654 private Set<Source> calculateRemovedSources() { 655 Set<Source> removed = new HashSet<>(); 656 for (String src : prev.sources().keySet()) { 657 if (now.sources().get(src) == null) { 658 removed.add(prev.sources().get(src)); 659 } 660 } 661 return removed; 662 } 663 664 /** 665 * Return those files belonging to now, but not prev. 666 */ 667 private Set<Source> calculateAddedSources() { 668 Set<Source> added = new HashSet<>(); 669 for (String src : now.sources().keySet()) { 670 if (prev.sources().get(src) == null) { 671 added.add(now.sources().get(src)); 672 } 673 } 674 return added; 675 } 676 677 /** 678 * Return those files where the timestamp is newer. 679 * If a source file timestamp suddenly is older than what is known 680 * about it in javac_state, then consider it modified, but print 681 * a warning! 682 */ 683 private Set<Source> calculateModifiedSources() { 684 Set<Source> modified = new HashSet<>(); 685 for (String src : now.sources().keySet()) { 686 Source n = now.sources().get(src); 687 Source t = prev.sources().get(src); 688 if (prev.sources().get(src) != null) { 689 if (t != null) { 690 if (n.lastModified() > t.lastModified()) { 691 modified.add(n); 692 } else if (n.lastModified() < t.lastModified()) { 693 modified.add(n); 694 Log.warn("The source file "+n.name()+" timestamp has moved backwards in time."); 695 } 696 } 697 } 698 } 699 return modified; 700 } 701 702 /** 703 * Recursively delete a directory and all its contents. 704 */ 705 private void deleteContents(File dir) { 706 if (dir != null && dir.exists()) { 707 for (File f : dir.listFiles()) { 708 if (f.isDirectory()) { 709 deleteContents(f); 710 } 711 if (!options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) { 712 Log.debug("Removing "+f.getAbsolutePath()); 713 f.delete(); 714 } 715 } 716 } 717 } 718 719 /** 720 * Run the copy translator only. 721 */ 722 public void performCopying(File binDir, Map<String,Transformer> suffixRules) { 723 Map<String,Transformer> sr = new HashMap<>(); 724 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) { 725 if (e.getValue().getClass().equals(CopyFile.class)) { 726 sr.put(e.getKey(), e.getValue()); 727 } 728 } 729 perform(null, binDir, sr); 730 } 731 732 /** 733 * Run all the translators that translate into java source code. 734 * I.e. all translators that are not copy nor compile_java_source. 735 */ 736 public void performTranslation(File gensrcDir, Map<String,Transformer> suffixRules) { 737 Map<String,Transformer> sr = new HashMap<>(); 738 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) { 739 Class<?> trClass = e.getValue().getClass(); 740 if (trClass == CompileJavaPackages.class || trClass == CopyFile.class) 741 continue; 742 743 sr.put(e.getKey(), e.getValue()); 744 } 745 perform(null, gensrcDir, sr); 746 } 747 748 /** 749 * Compile all the java sources. Return true, if it needs to be called again! 750 */ 751 public boolean performJavaCompilations(CompilationService sjavac, 752 Options args, 753 Set<String> recentlyCompiled, 754 boolean[] rcValue) { 755 Map<String,Transformer> suffixRules = new HashMap<>(); 756 suffixRules.put(".java", compileJavaPackages); 757 compileJavaPackages.setExtra(args); 758 rcValue[0] = perform(sjavac, binDir, suffixRules); 759 recentlyCompiled.addAll(taintedPackages()); 760 clearTaintedPackages(); 761 boolean again = !packagesWithChangedPublicApis.isEmpty(); 762 taintPackagesDependingOnChangedPackages(packagesWithChangedPublicApis, recentlyCompiled); 763 packagesWithChangedPublicApis = new HashSet<>(); 764 return again && rcValue[0]; 765 766 // TODO: Figure out why 'again' checks packagesWithChangedPublicAPis. 767 // (It shouldn't matter if packages had changed pub apis as long as no 768 // one depends on them. Wouldn't it make more sense to let 'again' 769 // depend on taintedPackages?) 770 } 771 772 /** 773 * Store the source into the set of sources belonging to the given transform. 774 */ 775 private void addFileToTransform(Map<Transformer,Map<String,Set<URI>>> gs, Transformer t, Source s) { 776 Map<String,Set<URI>> fs = gs.get(t); 777 if (fs == null) { 778 fs = new HashMap<>(); 779 gs.put(t, fs); 780 } 781 Set<URI> ss = fs.get(s.pkg().name()); 782 if (ss == null) { 783 ss = new HashSet<>(); 784 fs.put(s.pkg().name(), ss); 785 } 786 ss.add(s.file().toURI()); 787 } 788 789 /** 790 * For all packages, find all sources belonging to the package, group the sources 791 * based on their transformers and apply the transformers on each source code group. 792 */ 793 private boolean perform(CompilationService sjavac, 794 File outputDir, 795 Map<String,Transformer> suffixRules) { 796 boolean rc = true; 797 // Group sources based on transforms. A source file can only belong to a single transform. 798 Map<Transformer,Map<String,Set<URI>>> groupedSources = new HashMap<>(); 799 for (Source src : now.sources().values()) { 800 Transformer t = suffixRules.get(src.suffix()); 801 if (t != null) { 802 if (taintedPackages.contains(src.pkg().name()) && !src.isLinkedOnly()) { 803 addFileToTransform(groupedSources, t, src); 804 } 805 } 806 } 807 // Go through the transforms and transform them. 808 for (Map.Entry<Transformer, Map<String, Set<URI>>> e : groupedSources.entrySet()) { 809 Transformer t = e.getKey(); 810 Map<String, Set<URI>> srcs = e.getValue(); 811 // These maps need to be synchronized since multiple threads will be 812 // writing results into them. 813 Map<String, Set<URI>> packageArtifacts = Collections.synchronizedMap(new HashMap<>()); 814 Map<String, Map<String, Set<String>>> packageDependencies = Collections.synchronizedMap(new HashMap<>()); 815 Map<String, Map<String, Set<String>>> packageCpDependencies = Collections.synchronizedMap(new HashMap<>()); 816 Map<String, PubApi> packagePublicApis = Collections.synchronizedMap(new HashMap<>()); 817 Map<String, PubApi> dependencyPublicApis = Collections.synchronizedMap(new HashMap<>()); 818 819 boolean r = t.transform(sjavac, 820 srcs, 821 visibleSrcs, 822 visibleClasses, 823 prev.dependents(), 824 outputDir.toURI(), 825 packageArtifacts, 826 packageDependencies, 827 packageCpDependencies, 828 packagePublicApis, 829 dependencyPublicApis, 830 0, 831 isIncremental(), 832 numCores, 833 out, 834 err); 835 if (!r) 836 rc = false; 837 838 for (String p : srcs.keySet()) { 839 recompiledPackages.add(p); 840 } 841 // The transform is done! Extract all the artifacts and store the info into the Package objects. 842 for (Map.Entry<String, Set<URI>> a : packageArtifacts.entrySet()) { 843 Module mnow = now.findModuleFromPackageName(a.getKey()); 844 mnow.addArtifacts(a.getKey(), a.getValue()); 845 } 846 // Extract all the dependencies and store the info into the Package objects. 847 for (Map.Entry<String, Map<String, Set<String>>> a : packageDependencies.entrySet()) { 848 Map<String, Set<String>> deps = a.getValue(); 849 Module mnow = now.findModuleFromPackageName(a.getKey()); 850 mnow.setDependencies(a.getKey(), deps, false); 851 } 852 for (Map.Entry<String, Map<String, Set<String>>> a : packageCpDependencies.entrySet()) { 853 Map<String, Set<String>> deps = a.getValue(); 854 Module mnow = now.findModuleFromPackageName(a.getKey()); 855 mnow.setDependencies(a.getKey(), deps, true); 856 } 857 858 // This map contains the public api of the types that this 859 // compilation depended upon. This means that it may not contain 860 // full packages. In other words, we shouldn't remove knowledge of 861 // public apis but merge these with what we already have. 862 for (Map.Entry<String, PubApi> a : dependencyPublicApis.entrySet()) { 863 String pkg = a.getKey(); 864 PubApi packagePartialPubApi = a.getValue(); 865 Package pkgNow = now.findModuleFromPackageName(pkg).lookupPackage(pkg); 866 PubApi currentPubApi = pkgNow.getPubApi(); 867 PubApi newPubApi = PubApi.mergeTypes(currentPubApi, packagePartialPubApi); 868 pkgNow.setPubapi(newPubApi); 869 870 // See JDK-8071904 871 if (now.packages().containsKey(pkg)) 872 now.packages().get(pkg).setPubapi(newPubApi); 873 else 874 now.packages().put(pkg, pkgNow); 875 } 876 877 // The packagePublicApis cover entire packages (since sjavac compiles 878 // stuff on package level). This means that if a type is missing 879 // in the public api of a given package, it means that it has been 880 // removed. In other words, we should *set* the pubapi to whatever 881 // this map contains, and not merge it with what we already have. 882 for (Map.Entry<String, PubApi> a : packagePublicApis.entrySet()) { 883 String pkg = a.getKey(); 884 PubApi newPubApi = a.getValue(); 885 Module mprev = prev.findModuleFromPackageName(pkg); 886 Module mnow = now.findModuleFromPackageName(pkg); 887 mnow.setPubapi(pkg, newPubApi); 888 if (mprev.hasPubapiChanged(pkg, newPubApi)) { 889 // Aha! The pubapi of this package has changed! 890 // It can also be a new compile from scratch. 891 if (mprev.lookupPackage(pkg).existsInJavacState()) { 892 // This is an incremental compile! The pubapi 893 // did change. Trigger recompilation of dependents. 894 packagesWithChangedPublicApis.add(pkg); 895 Log.info("The API of " + Util.justPackageName(pkg) + " has changed!"); 896 } 897 } 898 } 899 } 900 return rc; 901 } 902 903 /** 904 * Utility method to recursively find all files below a directory. 905 */ 906 private static Set<File> findAllFiles(File dir) { 907 Set<File> foundFiles = new HashSet<>(); 908 if (dir == null) { 909 return foundFiles; 910 } 911 recurse(dir, foundFiles); 912 return foundFiles; 913 } 914 915 private static void recurse(File dir, Set<File> foundFiles) { 916 for (File f : dir.listFiles()) { 917 if (f.isFile()) { 918 foundFiles.add(f); 919 } else if (f.isDirectory()) { 920 recurse(f, foundFiles); 921 } 922 } 923 } 924 925 /** 926 * Compare the calculate source list, with an explicit list, usually 927 * supplied from the makefile. Used to detect bugs where the makefile and 928 * sjavac have different opinions on which files should be compiled. 929 */ 930 public void compareWithMakefileList(File makefileSourceList) 931 throws ProblemException { 932 // If we are building on win32 using for example cygwin the paths in the 933 // makefile source list 934 // might be /cygdrive/c/.... which does not match c:\.... 935 // We need to adjust our calculated sources to be identical, if 936 // necessary. 937 boolean mightNeedRewriting = File.pathSeparatorChar == ';'; 938 939 if (makefileSourceList == null) 940 return; 941 942 Set<String> calculatedSources = new HashSet<>(); 943 Set<String> listedSources = new HashSet<>(); 944 945 // Create a set of filenames with full paths. 946 for (Source s : now.sources().values()) { 947 // Don't include link only sources when comparing sources to compile 948 if (!s.isLinkedOnly()) { 949 String path = s.file().getPath(); 950 if (mightNeedRewriting) 951 path = Util.normalizeDriveLetter(path); 952 calculatedSources.add(path); 953 } 954 } 955 // Read in the file and create another set of filenames with full paths. 956 try { 957 BufferedReader in = new BufferedReader(new FileReader(makefileSourceList)); 958 for (;;) { 959 String l = in.readLine(); 960 if (l==null) break; 961 l = l.trim(); 962 if (mightNeedRewriting) { 963 if (l.indexOf(":") == 1 && l.indexOf("\\") == 2) { 964 // Everything a-ok, the format is already C:\foo\bar 965 } else if (l.indexOf(":") == 1 && l.indexOf("/") == 2) { 966 // The format is C:/foo/bar, rewrite into the above format. 967 l = l.replaceAll("/","\\\\"); 968 } else if (l.charAt(0) == '/' && l.indexOf("/",1) != -1) { 969 // The format might be: /cygdrive/c/foo/bar, rewrite into the above format. 970 // Do not hardcode the name cygdrive here. 971 int slash = l.indexOf("/",1); 972 l = l.replaceAll("/","\\\\"); 973 l = ""+l.charAt(slash+1)+":"+l.substring(slash+2); 974 } 975 if (Character.isLowerCase(l.charAt(0))) { 976 l = Character.toUpperCase(l.charAt(0))+l.substring(1); 977 } 978 } 979 listedSources.add(l); 980 } 981 } catch (FileNotFoundException | NoSuchFileException e) { 982 throw new ProblemException("Could not open "+makefileSourceList.getPath()+" since it does not exist!"); 983 } catch (IOException e) { 984 throw new ProblemException("Could not read "+makefileSourceList.getPath()); 985 } 986 987 for (String s : listedSources) { 988 if (!calculatedSources.contains(s)) { 989 throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!"); 990 } 991 } 992 993 for (String s : calculatedSources) { 994 if (!listedSources.contains(s)) { 995 throw new ProblemException("The smart javac wrapper calculated source "+s+" was not listed by the makefiles!"); 996 } 997 } 998 } 999} 1000