DocCommentTester.java revision 3074:522e516b8a83
1/* 2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24import java.io.File; 25import java.io.FileWriter; 26import java.io.IOException; 27import java.io.PrintWriter; 28import java.io.StringWriter; 29import java.io.Writer; 30import java.text.BreakIterator; 31import java.util.ArrayList; 32import java.util.Arrays; 33import java.util.List; 34import java.util.Locale; 35import java.util.regex.Matcher; 36import java.util.regex.Pattern; 37import java.util.stream.Collectors; 38 39import javax.lang.model.element.Name; 40import javax.tools.JavaFileObject; 41import javax.tools.StandardJavaFileManager; 42 43import com.sun.source.doctree.*; 44import com.sun.source.tree.ClassTree; 45import com.sun.source.tree.CompilationUnitTree; 46import com.sun.source.tree.MethodTree; 47import com.sun.source.tree.Tree; 48import com.sun.source.tree.VariableTree; 49import com.sun.source.util.DocTreeScanner; 50import com.sun.source.util.DocTrees; 51import com.sun.source.util.JavacTask; 52import com.sun.source.util.TreePath; 53import com.sun.source.util.TreePathScanner; 54import com.sun.tools.javac.api.JavacTool; 55import com.sun.tools.javac.tree.DCTree; 56import com.sun.tools.javac.tree.DCTree.DCDocComment; 57import com.sun.tools.javac.tree.DCTree.DCErroneous; 58import com.sun.tools.javac.tree.DocPretty; 59 60public class DocCommentTester { 61 62 public static final String BI_MARKER = "BREAK_ITERATOR"; 63 public final boolean useBreakIterator; 64 65 public DocCommentTester(boolean useBreakIterator) { 66 this.useBreakIterator = useBreakIterator; 67 } 68 public static void main(String... args) throws Exception { 69 ArrayList<String> list = new ArrayList(Arrays.asList(args)); 70 if (!list.isEmpty() && "-useBreakIterator".equals(list.get(0))) { 71 list.remove(0); 72 new DocCommentTester(true).run(list); 73 } else { 74 new DocCommentTester(false).run(list); 75 } 76 } 77 78 public void run(List<String> args) throws Exception { 79 String testSrc = System.getProperty("test.src"); 80 81 List<File> files = args.stream() 82 .map(arg -> new File(testSrc, arg)) 83 .collect(Collectors.toList()); 84 85 JavacTool javac = JavacTool.create(); 86 StandardJavaFileManager fm = javac.getStandardFileManager(null, null, null); 87 88 Iterable<? extends JavaFileObject> fos = fm.getJavaFileObjectsFromFiles(files); 89 90 JavacTask t = javac.getTask(null, fm, null, null, null, fos); 91 final DocTrees trees = DocTrees.instance(t); 92 93 if (useBreakIterator) { 94 // BreakIterators are locale dependent wrt. behavior 95 trees.setBreakIterator(BreakIterator.getSentenceInstance(Locale.ENGLISH)); 96 } 97 98 final Checker[] checkers = { 99 new ASTChecker(this, trees), 100 new PosChecker(this, trees), 101 new PrettyChecker(this, trees) 102 }; 103 104 DeclScanner d = new DeclScanner() { 105 @Override 106 public Void visitCompilationUnit(CompilationUnitTree tree, Void ignore) { 107 for (Checker c: checkers) 108 c.visitCompilationUnit(tree); 109 return super.visitCompilationUnit(tree, ignore); 110 } 111 112 @Override 113 void visitDecl(Tree tree, Name name) { 114 TreePath path = getCurrentPath(); 115 String dc = trees.getDocComment(path); 116 if (dc != null) { 117 for (Checker c : checkers) { 118 try { 119 System.err.println(path.getLeaf().getKind() 120 + " " + name 121 + " " + c.getClass().getSimpleName()); 122 123 c.check(path, name); 124 125 System.err.println(); 126 } catch (Exception e) { 127 error("Exception " + e); 128 e.printStackTrace(System.err); 129 } 130 } 131 } 132 } 133 }; 134 135 Iterable<? extends CompilationUnitTree> units = t.parse(); 136 for (CompilationUnitTree unit: units) { 137 d.scan(unit, null); 138 } 139 140 if (errors > 0) 141 throw new Exception(errors + " errors occurred"); 142 } 143 144 static abstract class DeclScanner extends TreePathScanner<Void, Void> { 145 abstract void visitDecl(Tree tree, Name name); 146 147 @Override 148 public Void visitClass(ClassTree tree, Void ignore) { 149 super.visitClass(tree, ignore); 150 visitDecl(tree, tree.getSimpleName()); 151 return null; 152 } 153 154 @Override 155 public Void visitMethod(MethodTree tree, Void ignore) { 156 super.visitMethod(tree, ignore); 157 visitDecl(tree, tree.getName()); 158 return null; 159 } 160 161 @Override 162 public Void visitVariable(VariableTree tree, Void ignore) { 163 super.visitVariable(tree, ignore); 164 visitDecl(tree, tree.getName()); 165 return null; 166 } 167 } 168 169 /** 170 * Base class for checkers to check the doc comment on a declaration 171 * (when present.) 172 */ 173 abstract class Checker { 174 final DocTrees trees; 175 176 Checker(DocTrees trees) { 177 this.trees = trees; 178 } 179 180 void visitCompilationUnit(CompilationUnitTree tree) { } 181 182 abstract void check(TreePath tree, Name name) throws Exception; 183 184 void error(String msg) { 185 DocCommentTester.this.error(msg); 186 } 187 } 188 189 void error(String msg) { 190 System.err.println("Error: " + msg); 191 errors++; 192 } 193 194 int errors; 195 196 /** 197 * Verify the structure of the DocTree AST by comparing it against golden text. 198 */ 199 static class ASTChecker extends Checker { 200 static final String NEWLINE = System.getProperty("line.separator"); 201 Printer printer = new Printer(); 202 String source; 203 DocCommentTester test; 204 205 ASTChecker(DocCommentTester test, DocTrees t) { 206 test.super(t); 207 this.test = test; 208 } 209 210 @Override 211 void visitCompilationUnit(CompilationUnitTree tree) { 212 try { 213 source = tree.getSourceFile().getCharContent(true).toString(); 214 } catch (IOException e) { 215 source = ""; 216 } 217 } 218 219 void check(TreePath path, Name name) { 220 StringWriter out = new StringWriter(); 221 DocCommentTree dc = trees.getDocCommentTree(path); 222 printer.print(dc, out); 223 out.flush(); 224 String found = out.toString().replace(NEWLINE, "\n"); 225 226 /* 227 * Look for the first block comment after the first occurrence 228 * of name, noting that, block comments with BI_MARKER may 229 * very well be present. 230 */ 231 int start = test.useBreakIterator 232 ? source.indexOf("\n/*\n" + BI_MARKER + "\n", findName(source, name)) 233 : source.indexOf("\n/*\n", findName(source, name)); 234 int end = source.indexOf("\n*/\n", start); 235 int startlen = start + (test.useBreakIterator ? BI_MARKER.length() + 1 : 0) + 4; 236 String expect = source.substring(startlen, end + 1); 237 if (!found.equals(expect)) { 238 if (test.useBreakIterator) { 239 System.err.println("Using BreakIterator"); 240 } 241 System.err.println("Expect:\n" + expect); 242 System.err.println("Found:\n" + found); 243 error("AST mismatch for " + name); 244 } 245 } 246 247 /** 248 * This main program is to set up the golden comments used by this 249 * checker. 250 * Usage: 251 * java DocCommentTester$ASTChecker -o dir file... 252 * The given files are written to the output directory with their 253 * golden comments updated. The intent is that the files should 254 * then be compared with the originals, e.g. with meld, and if the 255 * changes are approved, the new files can be used to replace the old. 256 */ 257 public static void main(String... args) throws Exception { 258 List<File> files = new ArrayList<File>(); 259 File o = null; 260 for (int i = 0; i < args.length; i++) { 261 String arg = args[i]; 262 if (arg.equals("-o")) 263 o = new File(args[++i]); 264 else if (arg.startsWith("-")) 265 throw new IllegalArgumentException(arg); 266 else { 267 files.add(new File(arg)); 268 } 269 } 270 271 if (o == null) 272 throw new IllegalArgumentException("no output dir specified"); 273 final File outDir = o; 274 275 JavacTool javac = JavacTool.create(); 276 StandardJavaFileManager fm = javac.getStandardFileManager(null, null, null); 277 Iterable<? extends JavaFileObject> fos = fm.getJavaFileObjectsFromFiles(files); 278 279 JavacTask t = javac.getTask(null, fm, null, null, null, fos); 280 final DocTrees trees = DocTrees.instance(t); 281 282 DeclScanner d = new DeclScanner() { 283 Printer p = new Printer(); 284 String source; 285 286 @Override 287 public Void visitCompilationUnit(CompilationUnitTree tree, Void ignore) { 288 System.err.println("processing " + tree.getSourceFile().getName()); 289 try { 290 source = tree.getSourceFile().getCharContent(true).toString(); 291 } catch (IOException e) { 292 source = ""; 293 } 294 // remove existing gold by removing all block comments after the first '{'. 295 int start = source.indexOf("{"); 296 while ((start = source.indexOf("\n/*\n", start)) != -1) { 297 int end = source.indexOf("\n*/\n"); 298 source = source.substring(0, start + 1) + source.substring(end + 4); 299 } 300 301 // process decls in compilation unit 302 super.visitCompilationUnit(tree, ignore); 303 304 // write the modified source 305 File f = new File(tree.getSourceFile().getName()); 306 File outFile = new File(outDir, f.getName()); 307 try { 308 FileWriter out = new FileWriter(outFile); 309 try { 310 out.write(source); 311 } finally { 312 out.close(); 313 } 314 } catch (IOException e) { 315 System.err.println("Can't write " + tree.getSourceFile().getName() 316 + " to " + outFile + ": " + e); 317 } 318 return null; 319 } 320 321 @Override 322 void visitDecl(Tree tree, Name name) { 323 DocTree dc = trees.getDocCommentTree(getCurrentPath()); 324 if (dc != null) { 325 StringWriter out = new StringWriter(); 326 p.print(dc, out); 327 String found = out.toString(); 328 329 // Look for the empty line after the first occurrence of name 330 int pos = source.indexOf("\n\n", findName(source, name)); 331 332 // Insert the golden comment 333 source = source.substring(0, pos) 334 + "\n/*\n" 335 + found 336 + "*/" 337 + source.substring(pos); 338 } 339 } 340 341 }; 342 343 Iterable<? extends CompilationUnitTree> units = t.parse(); 344 for (CompilationUnitTree unit: units) { 345 d.scan(unit, null); 346 } 347 } 348 349 static int findName(String source, Name name) { 350 Pattern p = Pattern.compile("\\s" + name + "[(;]"); 351 Matcher m = p.matcher(source); 352 if (!m.find()) 353 throw new Error("cannot find " + name); 354 return m.start(); 355 } 356 357 static class Printer implements DocTreeVisitor<Void, Void> { 358 PrintWriter out; 359 360 void print(DocTree tree, Writer out) { 361 this.out = (out instanceof PrintWriter) 362 ? (PrintWriter) out : new PrintWriter(out); 363 tree.accept(this, null); 364 this.out.flush(); 365 } 366 367 public Void visitAttribute(AttributeTree node, Void p) { 368 header(node); 369 indent(+1); 370 print("name", node.getName().toString()); 371 print("vkind", node.getValueKind().toString()); 372 print("value", node.getValue()); 373 indent(-1); 374 indent(); 375 out.println("]"); 376 return null; 377 } 378 379 public Void visitAuthor(AuthorTree node, Void p) { 380 header(node); 381 indent(+1); 382 print("name", node.getName()); 383 indent(-1); 384 indent(); 385 out.println("]"); 386 return null; 387 } 388 389 public Void visitComment(CommentTree node, Void p) { 390 header(node, compress(node.getBody())); 391 return null; 392 } 393 394 public Void visitDeprecated(DeprecatedTree node, Void p) { 395 header(node); 396 indent(+1); 397 print("body", node.getBody()); 398 indent(-1); 399 indent(); 400 out.println("]"); 401 return null; 402 } 403 404 public Void visitDocComment(DocCommentTree node, Void p) { 405 header(node); 406 indent(+1); 407 print("firstSentence", node.getFirstSentence()); 408 print("body", node.getBody()); 409 print("block tags", node.getBlockTags()); 410 indent(-1); 411 indent(); 412 out.println("]"); 413 return null; 414 } 415 416 public Void visitDocRoot(DocRootTree node, Void p) { 417 header(node, ""); 418 return null; 419 } 420 421 public Void visitEndElement(EndElementTree node, Void p) { 422 header(node, node.getName().toString()); 423 return null; 424 } 425 426 public Void visitEntity(EntityTree node, Void p) { 427 header(node, node.getName().toString()); 428 return null; 429 } 430 431 public Void visitErroneous(ErroneousTree node, Void p) { 432 header(node); 433 indent(+1); 434 print("code", ((DCErroneous) node).diag.getCode()); 435 print("body", compress(node.getBody())); 436 indent(-1); 437 indent(); 438 out.println("]"); 439 return null; 440 } 441 442 public Void visitIdentifier(IdentifierTree node, Void p) { 443 header(node, compress(node.getName().toString())); 444 return null; 445 } 446 447 public Void visitInheritDoc(InheritDocTree node, Void p) { 448 header(node, ""); 449 return null; 450 } 451 452 public Void visitLink(LinkTree node, Void p) { 453 header(node); 454 indent(+1); 455 print("reference", node.getReference()); 456 print("body", node.getLabel()); 457 indent(-1); 458 indent(); 459 out.println("]"); 460 return null; 461 } 462 463 public Void visitLiteral(LiteralTree node, Void p) { 464 header(node, compress(node.getBody().getBody())); 465 return null; 466 } 467 468 public Void visitParam(ParamTree node, Void p) { 469 header(node); 470 indent(+1); 471 print("name", node.getName()); 472 print("description", node.getDescription()); 473 indent(-1); 474 indent(); 475 out.println("]"); 476 return null; 477 } 478 479 public Void visitReference(ReferenceTree node, Void p) { 480 header(node, compress(node.getSignature())); 481 return null; 482 } 483 484 public Void visitReturn(ReturnTree node, Void p) { 485 header(node); 486 indent(+1); 487 print("description", node.getDescription()); 488 indent(-1); 489 indent(); 490 out.println("]"); 491 return null; 492 } 493 494 public Void visitSee(SeeTree node, Void p) { 495 header(node); 496 indent(+1); 497 print("reference", node.getReference()); 498 indent(-1); 499 indent(); 500 out.println("]"); 501 return null; 502 } 503 504 public Void visitSerial(SerialTree node, Void p) { 505 header(node); 506 indent(+1); 507 print("description", node.getDescription()); 508 indent(-1); 509 indent(); 510 out.println("]"); 511 return null; 512 } 513 514 public Void visitSerialData(SerialDataTree node, Void p) { 515 header(node); 516 indent(+1); 517 print("description", node.getDescription()); 518 indent(-1); 519 indent(); 520 out.println("]"); 521 return null; 522 } 523 524 public Void visitSerialField(SerialFieldTree node, Void p) { 525 header(node); 526 indent(+1); 527 print("name", node.getName()); 528 print("type", node.getType()); 529 print("description", node.getDescription()); 530 indent(-1); 531 indent(); 532 out.println("]"); 533 return null; 534 } 535 536 public Void visitSince(SinceTree node, Void p) { 537 header(node); 538 indent(+1); 539 print("body", node.getBody()); 540 indent(-1); 541 indent(); 542 out.println("]"); 543 return null; 544 } 545 546 public Void visitStartElement(StartElementTree node, Void p) { 547 header(node); 548 indent(+1); 549 indent(); 550 out.println("name:" + node.getName()); 551 print("attributes", node.getAttributes()); 552 indent(-1); 553 indent(); 554 out.println("]"); 555 return null; 556 } 557 558 public Void visitText(TextTree node, Void p) { 559 header(node, compress(node.getBody())); 560 return null; 561 } 562 563 public Void visitThrows(ThrowsTree node, Void p) { 564 header(node); 565 indent(+1); 566 print("exceptionName", node.getExceptionName()); 567 print("description", node.getDescription()); 568 indent(-1); 569 indent(); 570 out.println("]"); 571 return null; 572 } 573 574 public Void visitUnknownBlockTag(UnknownBlockTagTree node, Void p) { 575 header(node); 576 indent(+1); 577 indent(); 578 out.println("tag:" + node.getTagName()); 579 print("content", node.getContent()); 580 indent(-1); 581 indent(); 582 out.println("]"); 583 return null; 584 } 585 586 public Void visitUnknownInlineTag(UnknownInlineTagTree node, Void p) { 587 header(node); 588 indent(+1); 589 indent(); 590 out.println("tag:" + node.getTagName()); 591 print("content", node.getContent()); 592 indent(-1); 593 indent(); 594 out.println("]"); 595 return null; 596 } 597 598 public Void visitValue(ValueTree node, Void p) { 599 header(node); 600 indent(+1); 601 print("reference", node.getReference()); 602 indent(-1); 603 indent(); 604 out.println("]"); 605 return null; 606 } 607 608 public Void visitVersion(VersionTree node, Void p) { 609 header(node); 610 indent(+1); 611 print("body", node.getBody()); 612 indent(-1); 613 indent(); 614 out.println("]"); 615 return null; 616 } 617 618 public Void visitOther(DocTree node, Void p) { 619 throw new UnsupportedOperationException("Not supported yet."); 620 } 621 622 void header(DocTree node) { 623 indent(); 624 out.println(simpleClassName(node) + "[" + node.getKind() + ", pos:" + ((DCTree) node).pos); 625 } 626 627 void header(DocTree node, String rest) { 628 indent(); 629 out.println(simpleClassName(node) + "[" + node.getKind() + ", pos:" + ((DCTree) node).pos 630 + (rest.isEmpty() ? "" : ", " + rest) 631 + "]"); 632 } 633 634 String simpleClassName(DocTree node) { 635 return node.getClass().getSimpleName().replaceAll("DC(.*)", "$1"); 636 } 637 638 void print(String name, DocTree item) { 639 indent(); 640 if (item == null) 641 out.println(name + ": null"); 642 else { 643 out.println(name + ":"); 644 indent(+1); 645 item.accept(this, null); 646 indent(-1); 647 } 648 } 649 650 void print(String name, String s) { 651 indent(); 652 out.println(name + ": " + s); 653 } 654 655 void print(String name, List<? extends DocTree> list) { 656 indent(); 657 if (list == null) 658 out.println(name + ": null"); 659 else if (list.isEmpty()) 660 out.println(name + ": empty"); 661 else { 662 out.println(name + ": " + list.size()); 663 indent(+1); 664 for (DocTree tree: list) { 665 tree.accept(this, null); 666 } 667 indent(-1); 668 } 669 } 670 671 int indent = 0; 672 673 void indent() { 674 for (int i = 0; i < indent; i++) { 675 out.print(" "); 676 } 677 } 678 679 void indent(int n) { 680 indent += n; 681 } 682 683 String compress(String s) { 684 s = s.replace("\n", "|").replace(" ", "_"); 685 return (s.length() < 32) 686 ? s 687 : s.substring(0, 16) + "..." + s.substring(16); 688 } 689 690 String quote(String s) { 691 if (s.contains("\"")) 692 return "'" + s + "'"; 693 else if (s.contains("'") || s.contains(" ")) 694 return '"' + s + '"'; 695 else 696 return s; 697 } 698 } 699 } 700 701 /** 702 * Verify the reported tree positions by comparing the characters found 703 * at and after the reported position with the beginning of the pretty- 704 * printed text. 705 */ 706 static class PosChecker extends Checker { 707 PosChecker(DocCommentTester test, DocTrees t) { 708 test.super(t); 709 } 710 711 @Override 712 void check(TreePath path, Name name) throws Exception { 713 JavaFileObject fo = path.getCompilationUnit().getSourceFile(); 714 final CharSequence cs = fo.getCharContent(true); 715 716 final DCDocComment dc = (DCDocComment) trees.getDocCommentTree(path); 717 DCTree t = (DCTree) trees.getDocCommentTree(path); 718 719 DocTreeScanner scanner = new DocTreeScanner<Void,Void>() { 720 @Override 721 public Void scan(DocTree node, Void ignore) { 722 if (node != null) { 723 try { 724 String expect = getExpectText(node); 725 long pos = ((DCTree) node).getSourcePosition(dc); 726 String found = getFoundText(cs, (int) pos, expect.length()); 727 if (!found.equals(expect)) { 728 System.err.println("expect: " + expect); 729 System.err.println("found: " + found); 730 error("mismatch"); 731 } 732 733 } catch (StringIndexOutOfBoundsException e) { 734 error(node.getClass() + ": " + e.toString()); 735 e.printStackTrace(); 736 } 737 } 738 return super.scan(node, ignore); 739 } 740 }; 741 742 scanner.scan(t, null); 743 } 744 745 String getExpectText(DocTree t) { 746 StringWriter sw = new StringWriter(); 747 DocPretty p = new DocPretty(sw); 748 try { p.print(t); } catch (IOException never) { } 749 String s = sw.toString(); 750 if (s.length() <= 1) 751 return s; 752 int ws = s.replaceAll("\\s+", " ").indexOf(" "); 753 if (ws != -1) s = s.substring(0, ws); 754 return (s.length() < 5) ? s : s.substring(0, 5); 755 } 756 757 String getFoundText(CharSequence cs, int pos, int len) { 758 return (pos == -1) ? "" : cs.subSequence(pos, Math.min(pos + len, cs.length())).toString(); 759 } 760 } 761 762 /** 763 * Verify the pretty printed text against a normalized form of the 764 * original doc comment. 765 */ 766 static class PrettyChecker extends Checker { 767 768 PrettyChecker(DocCommentTester test, DocTrees t) { 769 test.super(t); 770 } 771 772 @Override 773 void check(TreePath path, Name name) throws Exception { 774 String raw = trees.getDocComment(path); 775 String normRaw = normalize(raw); 776 777 StringWriter out = new StringWriter(); 778 DocPretty dp = new DocPretty(out); 779 dp.print(trees.getDocCommentTree(path)); 780 String pretty = out.toString(); 781 782 if (!pretty.equals(normRaw)) { 783 error("mismatch"); 784 System.err.println("*** expected:"); 785 System.err.println(normRaw.replace(" ", "_")); 786 System.err.println("*** found:"); 787 System.err.println(pretty.replace(" ", "_")); 788 // throw new Error(); 789 } 790 } 791 792 /** 793 * Normalize white space in places where the tree does not preserve it. 794 */ 795 String normalize(String s) { 796 s = s.trim() 797 .replaceFirst("\\.\\s*\\n *@", ".\n@") 798 .replaceAll("\\{@docRoot\\s+\\}", "{@docRoot}") 799 .replaceAll("\\{@inheritDoc\\s+\\}", "{@inheritDoc}") 800 .replaceAll("(\\{@value\\s+[^}]+)\\s+(\\})", "$1$2") 801 .replaceAll("\n[ \t]+@", "\n@") 802 .replaceAll("(\\{@code)(\\x20)(\\s+.*)", "$1$3"); 803 return s; 804 } 805 } 806} 807 808