javadoc/lib/JavadocTester.java

/*
 * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

import com.sun.javadoc.*;
import java.util.*;
import java.io.*;


/**
 * Runs javadoc and then runs regression tests on the resulting output.
 * This class currently contains three tests:
 * <ul>
 * <li> String search: Reads each file, complete with newlines,
 *      into a string.  Lets you search for strings that contain
 *      newlines.  String matching is case-sensitive.
 *      You can run javadoc multiple times with different arguments,
 *      generating output into different destination directories, and
 *      then perform a different array of tests on each one.
 *      To do this, the run method accepts a test array for testing
 *      that a string is found, and a negated test array for testing
 *      that a string is not found.
 * <li> Run diffs: Iterate through the list of given file pairs
 *      and diff the pairs.
 * <li> Check exit code: Check the exit code of Javadoc and
 *      record whether the test passed or failed.
 * </ul>
 *
 * @author Doug Kramer
 * @author Jamie Ho
 * @since 1.4.2
 */
public abstract class JavadocTester {

    protected static final String NL = System.getProperty("line.separator");
    protected static final String FS = System.getProperty("file.separator");

    protected static final String SRC_DIR = System.getProperty("test.src", ".");
    protected static final String JAVA_VERSION = System.getProperty("java.version");
    protected static final String[][] NO_TEST = new String[][] {};
    protected static final String[] NO_FILE_TEST = new String[] {};

    /**
     * Use this as the file name in the test array when you want to search
     * for a string in the error output.
     */
    public static final String ERROR_OUTPUT = "ERROR_OUTPUT";

    /**
     * Use this as the file name in the test array when you want to search
     * for a string in the notice output.
     */
    public static final String NOTICE_OUTPUT = "NOTICE_OUTPUT";

    /**
     * Use this as the file name in the test array when you want to search
     * for a string in the warning output.
     */
    public static final String WARNING_OUTPUT = "WARNING_OUTPUT";

    /**
     * Use this as the file name in the test array when you want to search
     * for a string in standard output.
     */
    public static final String STANDARD_OUTPUT = "STANDARD_OUTPUT";

    /**
     * The default doclet.
     */
    public static final String DEFAULT_DOCLET_CLASS = "com.sun.tools.doclets.formats.html.HtmlDoclet";
    public static final String DEFAULT_DOCLET_CLASS_OLD = "com.sun.tools.doclets.standard.Standard";

    /**
     * The writer to write error messages.
     */
    public StringWriter errors;

    /**
     * The writer to write notices.
     */
    public StringWriter notices;

    /**
     * The writer to write warnings.
     */
    public StringWriter warnings;

    /**
     * The buffer of warning output..
     */
    public StringBuffer standardOut;

    /**
     * The current subtest number.
     */
    private static int numTestsRun = 0;

    /**
     * The number of subtests passed.
     */
    private static int numTestsPassed = 0;

    /**
     * The current run of javadoc
     */
    private static int javadocRunNum = 0;

    /**
     * Construct a JavadocTester.
     */
    public JavadocTester() {
    }

    /**
     * Return the bug id.
     * @return the bug id
     */
    public abstract String getBugId();

    /**
     * Return the name of the bug.
     * @return the name of the bug
     */
    public abstract String getBugName();

    /**
     * Execute the tests.
     *
     * @param tester           the tester to execute
     * @param args             the arguments to pass to Javadoc
     * @param testArray        the array of tests
     * @param negatedTestArray the array of negated tests
     * @return                 the return code for the execution of Javadoc
     */
    public static int run(JavadocTester tester, String[] args,
            String[][] testArray, String[][] negatedTestArray) {
        int returnCode = tester.runJavadoc(args);
        tester.runTestsOnHTML(testArray, negatedTestArray);
        return returnCode;
    }

    /**
     * Execute the tests.
     *
     * @param tester               the tester to execute
     * @param args                 the arguments to pass to Javadoc
     * @param testArray            the array of tests
     * @param negatedTestArray     the array of negated tests
     * @param fileTestArray        the array of file tests
     * @param negatedFileTestArray the array of negated file tests
     * @return                     the return code for the execution of Javadoc
     */
    public static int run(JavadocTester tester, String[] args,
            String[][] testArray, String[][] negatedTestArray, String[] fileTestArray,
            String[] negatedFileTestArray) {
        int returnCode = tester.runJavadoc(args);
        tester.runTestsOnHTML(testArray, negatedTestArray);
        tester.runTestsOnFile(fileTestArray, negatedFileTestArray);
        return returnCode;
    }

    /**
     * Execute Javadoc using the default doclet.
     *
     * @param args  the arguments to pass to Javadoc
     * @return      the return code from the execution of Javadoc
     */
    public int runJavadoc(String[] args) {
        float javaVersion = Float.parseFloat(JAVA_VERSION.substring(0,3));
        String docletClass = javaVersion < 1.5 ?
            DEFAULT_DOCLET_CLASS_OLD : DEFAULT_DOCLET_CLASS;
        return runJavadoc(docletClass, args);
    }


    /**
     * Execute Javadoc.
     *
     * @param docletClass the doclet being tested.
     * @param args  the arguments to pass to Javadoc
     * @return      the return code from the execution of Javadoc
     */
    public int runJavadoc(String docletClass, String[] args) {
        javadocRunNum++;
        if (javadocRunNum == 1) {
            System.out.println("\n" + "Running javadoc...");
        } else {
            System.out.println("\n" + "Running javadoc (run "
                                    + javadocRunNum + ")...");
        }
        initOutputBuffers();

        ByteArrayOutputStream stdout = new ByteArrayOutputStream();
        PrintStream prevOut = System.out;
        System.setOut(new PrintStream(stdout));

        ByteArrayOutputStream stderr = new ByteArrayOutputStream();
        PrintStream prevErr = System.err;
        System.setErr(new PrintStream(stderr));

        int returnCode = com.sun.tools.javadoc.Main.execute(
                getBugName(),
                new PrintWriter(errors, true),
                new PrintWriter(warnings, true),
                new PrintWriter(notices, true),
                docletClass,
                getClass().getClassLoader(),
                args);
        System.setOut(prevOut);
        standardOut = new StringBuffer(stdout.toString());
        System.setErr(prevErr);
        errors.write(NL + stderr.toString());

        printJavadocOutput();
        return returnCode;
    }

    /**
     * Create new string writer buffers
     */
    private void initOutputBuffers() {
        errors   = new StringWriter();
        notices  = new StringWriter();
        warnings = new StringWriter();
    }

    /**
     * Run array of tests on the resulting HTML.
     * This method accepts a testArray for testing that a string is found
     * and a negatedTestArray for testing that a string is not found.
     *
     * @param testArray         the array of tests
     * @param negatedTestArray  the array of negated tests
     */
    public void runTestsOnHTML(String[][] testArray, String[][] negatedTestArray) {
        runTestsOnHTML(testArray, false);
        runTestsOnHTML(negatedTestArray, true);
    }

    /**
     * Run array of tests on the generated files.
     * This method accepts a fileTestArray for testing if a file is generated
     * and a negatedFileTestArray for testing if a file is not found.
     *
     * @param testArray         the array of file tests
     * @param negatedTestArray  the array of negated file tests
     */
    public void runTestsOnFile(String[] fileTestArray, String[] negatedFileTestArray) {
        runTestsOnFile(fileTestArray, false);
        runTestsOnFile(negatedFileTestArray, true);
    }

    /**
     * Run the array of tests on the resulting HTML.
     *
     * @param testArray the array of tests
     * @param isNegated true if test is negated; false otherwise
     */
    private void runTestsOnHTML(String[][] testArray , boolean isNegated) {
        for (int i = 0; i < testArray.length; i++) {

            numTestsRun++;

            System.out.print("Running subtest #" + numTestsRun + "... ");

            // Get string to find
            String stringToFind = testArray[i][1];

            // Read contents of file into a string
            String fileString;
            try {
                fileString = readFileToString(testArray[i][0]);
            } catch (Error e) {
                if (isNegated) {
                  System.out.println( "FAILED" + "\n"
                                    + "for bug " + getBugId()
                                    + " (" + getBugName() + ") "
                                    + "due to "
                                    + e + "\n");
                  continue;
                }
                throw e;
            }
            // Find string in file's contents
            boolean isFound = findString(fileString, stringToFind);
            if ((isNegated && !isFound) || (!isNegated && isFound) ) {
                numTestsPassed += 1;
                System.out.println( "Passed" + "\n"
                                    + (isNegated ? "not found:" : "found:") + "\n"
                                    + stringToFind + " in " + testArray[i][0] + "\n");
            } else {
                System.out.println( "FAILED" + "\n"
                                    + "for bug " + getBugId()
                                    + " (" + getBugName() + ")" + "\n"
                                    + "when searching for:" + "\n"
                                    + stringToFind
                                    + " in " + testArray[i][0] + "\n");
            }
        }
    }

    /**
     * Run the array of file tests on the generated files.
     *
     * @param testArray the array of file tests
     * @param isNegated true if test is negated; false otherwise
     */
    private void runTestsOnFile(String[] testArray, boolean isNegated) {
        String fileName;
        String failedString;
        String passedString;
        for (int i = 0; i < testArray.length; i++) {
            numTestsRun++;
            fileName = testArray[i];
            failedString = "FAILED" + "\n"
                    + "for bug " + getBugId() + " (" + getBugName() + ") "
                    + "file (" + fileName + ") found" + "\n";
            passedString = "Passed" + "\n" +
                        "file (" + fileName + ") not found" + "\n";
            System.out.print("Running subtest #" + numTestsRun + "... ");
            try {
                File file = new File(fileName);
                if ((file.exists() && !isNegated) || (!file.exists() && isNegated)) {
                    numTestsPassed += 1;
                    System.out.println(passedString);
                } else {
                    System.out.println(failedString);
                }
            } catch (Error e) {
                System.err.println(e);
            }
        }
    }

    /**
     * Iterate through the list of given file pairs and diff each file.
     *
     * @param filePairs the pairs of files to diff.
     * @throws an Error is thrown if any differences are found between
     * file pairs.
     */
    public void runDiffs(String[][] filePairs) throws Error {
        runDiffs(filePairs, true);
    }

    /**
     * Iterate through the list of given file pairs and diff each file.
     *
     * @param filePairs the pairs of files to diff.
     * @param throwErrorIFNoMatch flag to indicate whether or not to throw
     * an error if the files do not match.
     *
     * @throws an Error is thrown if any differences are found between
     * file pairs and throwErrorIFNoMatch is true.
     */
    public void runDiffs(String[][] filePairs, boolean throwErrorIfNoMatch) throws Error {
        for (int i = 0; i < filePairs.length; i++) {
            diff(filePairs[i][0], filePairs[i][1], throwErrorIfNoMatch);
        }
    }

    /**
     * Check the exit code of Javadoc and record whether the test passed
     * or failed.
     *
     * @param expectedExitCode The exit code that is required for the test
     * to pass.
     * @param actualExitCode The actual exit code from the previous run of
     * Javadoc.
     */
    public void checkExitCode(int expectedExitCode, int actualExitCode) {
        numTestsRun++;
        if (expectedExitCode == actualExitCode) {
            System.out.println( "Passed" + "\n" + " got return code " +
                actualExitCode);
            numTestsPassed++;
        } else {
            System.out.println( "FAILED" + "\n" + "for bug " + getBugId()
                + " (" + getBugName() + ")" + "\n" + "Expected return code " +
                expectedExitCode + " but got " + actualExitCode);
        }
    }

    /**
     * Print a summary of the test results.
     */
    protected void printSummary() {
        if ( numTestsRun != 0 && numTestsPassed == numTestsRun ) {
            // Test passed
            System.out.println("\n" + "All " + numTestsPassed
                                             + " subtests passed");
        } else {
            // Test failed
            throw new Error("\n" + (numTestsRun - numTestsPassed)
                                    + " of " + (numTestsRun)
                                    + " subtests failed for bug " + getBugId()
                                    + " (" + getBugName() + ")" + "\n");
        }
    }

    /**
     * Print the output stored in the buffers.
     */
    protected void printJavadocOutput() {
        System.out.println(STANDARD_OUTPUT + " : \n" + getStandardOutput());
        System.err.println(ERROR_OUTPUT + " : \n" + getErrorOutput());
        System.err.println(WARNING_OUTPUT + " : \n" + getWarningOutput());
        System.out.println(NOTICE_OUTPUT + " : \n" + getNoticeOutput());
    }

    /**
     * Read the file and return it as a string.
     *
     * @param fileName  the name of the file to read
     * @return          the file in string format
     */
    public String readFileToString(String fileName) throws Error {
        if (fileName.equals(ERROR_OUTPUT)) {
            return getErrorOutput();
        } else if (fileName.equals(NOTICE_OUTPUT)) {
            return getNoticeOutput();
        } else if (fileName.equals(WARNING_OUTPUT)) {
            return getWarningOutput();
        } else if (fileName.equals(STANDARD_OUTPUT)) {
            return getStandardOutput();
        }
        try {
            File file = new File(fileName);
            if ( !file.exists() ) {
                System.out.println("\n" + "FILE DOES NOT EXIST: " + fileName);
            }
            BufferedReader in = new BufferedReader(new FileReader(file));

            // Create an array of characters the size of the file
            char[] allChars = new char[(int)file.length()];

            // Read the characters into the allChars array
            in.read(allChars, 0, (int)file.length());
            in.close();

            // Convert to a string
            String allCharsString = new String(allChars);
            return allCharsString;
        } catch (FileNotFoundException e) {
            System.err.println(e);
            throw new Error("File not found: " + fileName);
        } catch (IOException e) {
            System.err.println(e);
            throw new Error("Error reading file: " + fileName);
        }
    }

    /**
     * Compare the two given files.
     *
     * @param file1 the first file to compare.
     * @param file2 the second file to compare.
     * @param throwErrorIFNoMatch flag to indicate whether or not to throw
     * an error if the files do not match.
     * @return true if the files are the same and false otherwise.
     */
    public boolean diff(String file1, String file2, boolean throwErrorIFNoMatch) throws Error {
        String file1Contents = readFileToString(file1);
        String file2Contents = readFileToString(file2);
        numTestsRun++;
        if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
            System.out.println("Diff successful: " + file1 + ", " + file2);
            numTestsPassed++;
            return true;
        } else if (throwErrorIFNoMatch) {
            throw new Error("Diff failed: " + file1 + ", " + file2);
        } else {
            return false;
        }
    }

    /**
     * Search for the string in the given file and return true
     * if the string was found.
     *
     * @param fileString    the contents of the file to search through
     * @param stringToFind  the string to search for
     * @return              true if the string was found
     */
    private boolean findString(String fileString, String stringToFind) {
        return fileString.contains(stringToFind.replace("\n", NL));
    }


    /**
     * Return the standard output.
     * @return the standard output
     */
    public String getStandardOutput() {
        return standardOut.toString();
    }

    /**
     * Return the error output.
     * @return the error output
     */
    public String getErrorOutput() {
        return errors.getBuffer().toString();
    }

    /**
     * Return the notice output.
     * @return the notice output
     */
    public String getNoticeOutput() {
        return notices.getBuffer().toString();
    }

    /**
     * Return the warning output.
     * @return the warning output
     */
    public String getWarningOutput() {
        return warnings.getBuffer().toString();
    }

    /**
     * A utility to copy a directory from one place to another.
     * We may possibly want to move this to our doclet toolkit in
     * the near future and maintain it from there.
     *
     * @param targetDir the directory to copy.
     * @param destDir the destination to copy the directory to.
     */
    public static void copyDir(String targetDir, String destDir) {
        if (targetDir.endsWith("SCCS")) {
            return;
        }
        try {
            File targetDirObj = new File(targetDir);
            File destDirParentObj = new File(destDir);
            File destDirObj = new File(destDirParentObj, targetDirObj.getName());
            if (! destDirParentObj.exists()) {
                destDirParentObj.mkdir();
            }
            if (! destDirObj.exists()) {
                destDirObj.mkdir();
            }
            String[] files = targetDirObj.list();
            for (int i = 0; i < files.length; i++) {
                File srcFile = new File(targetDirObj, files[i]);
                File destFile = new File(destDirObj, files[i]);
                if (srcFile.isFile()) {
                    System.out.println("Copying " + srcFile + " to " + destFile);
                        copyFile(destFile, srcFile);
                } else if(srcFile.isDirectory()) {
                    copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
                }
            }
        } catch (IOException exc) {
            throw new Error("Could not copy " + targetDir + " to " + destDir);
        }
    }

    /**
     * Copy source file to destination file.
     *
     * @throws SecurityException
     * @throws IOException
     */
    public static void copyFile(File destfile, File srcfile)
        throws IOException {
        byte[] bytearr = new byte[512];
        int len = 0;
        FileInputStream input = new FileInputStream(srcfile);
        File destDir = destfile.getParentFile();
        destDir.mkdirs();
        FileOutputStream output = new FileOutputStream(destfile);
        try {
            while ((len = input.read(bytearr)) != -1) {
                output.write(bytearr, 0, len);
            }
        } catch (FileNotFoundException exc) {
        } catch (SecurityException exc) {
        } finally {
            input.close();
            output.close();
        }
    }
}