lib/combo/ComboTestHelper.java

/*
 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

package combo;

import javax.tools.JavaCompiler;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Stack;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;


/**
 * An helper class for defining combinatorial (aka "combo" tests). A combo test is made up of one
 * or more 'dimensions' - each of which represent a different axis of the test space. For instance,
 * if we wanted to test class/interface declaration, one dimension could be the keyword used for
 * the declaration (i.e. 'class' vs. 'interface') while another dimension could be the class/interface
 * modifiers (i.e. 'public', 'pachake-private' etc.). A combo test consists in running a test instance
 * for each point in the test space; that is, for any combination of the combo test dimension:
 * <p>
 * 'public' 'class'
 * 'public' interface'
 * 'package-private' 'class'
 * 'package-private' 'interface'
 * ...
 * <p>
 * A new test instance {@link ComboInstance} is created, and executed, after its dimensions have been
 * initialized accordingly. Each instance can either pass, fail or throw an unexpected error; this helper
 * class defines several policies for how failures should be handled during a combo test execution
 * (i.e. should errors be ignored? Do we want the first failure to result in a failure of the whole
 * combo test?).
 * <p>
 * Additionally, this helper class allows to specify filter methods that can be used to throw out
 * illegal combinations of dimensions - for instance, in the example above, we might want to exclude
 * all combinations involving 'protected' and 'private' modifiers, which are disallowed for toplevel
 * declarations.
 * <p>
 * While combo tests can be used for a variety of workloads, typically their main task will consist
 * in performing some kind of javac compilation. For this purpose, this framework defines an optimized
 * javac context {@link ReusableContext} which can be shared across multiple combo instances,
 * when the framework detects it's safe to do so. This allows to reduce the overhead associated with
 * compiler initialization when the test space is big.
 */
public class ComboTestHelper<X extends ComboInstance<X>> {

    /** Failure mode. */
    FailMode failMode = FailMode.FAIL_FAST;

    /** Ignore mode. */
    IgnoreMode ignoreMode = IgnoreMode.IGNORE_NONE;

    /** Combo test instance filter. */
    Optional<Predicate<X>> optFilter = Optional.empty();

    /** Combo test dimensions. */
    List<DimensionInfo<?>> dimensionInfos = new ArrayList<>();

    /** Combo test stats. */
    Info info = new Info();

    /** Shared JavaCompiler used across all combo test instances. */
    JavaCompiler comp = ToolProvider.getSystemJavaCompiler();

    /** Shared file manager used across all combo test instances. */
    StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);

    /** Shared context used across all combo instances. */
    ReusableContext context = new ReusableContext();

    /**
     * Set failure mode for this combo test.
     */
    public ComboTestHelper<X> withFailMode(FailMode failMode) {
        this.failMode = failMode;
        return this;
    }

    /**
     * Set ignore mode for this combo test.
     */
    public ComboTestHelper<X> withIgnoreMode(IgnoreMode ignoreMode) {
        this.ignoreMode = ignoreMode;
        return this;
    }

    /**
     * Set a filter for combo test instances to be ignored.
     */
    public ComboTestHelper<X> withFilter(Predicate<X> filter) {
        optFilter = Optional.of(optFilter.map(filter::and).orElse(filter));
        return this;
    }

    /**
     * Adds a new dimension to this combo test, with a given name an array of values.
     */
    @SafeVarargs
    public final <D> ComboTestHelper<X> withDimension(String name, D... dims) {
        return withDimension(name, null, dims);
    }

    /**
     * Adds a new dimension to this combo test, with a given name, an array of values and a
     * coresponding setter to be called in order to set the dimension value on the combo test instance
     * (before test execution).
     */
    @SuppressWarnings("unchecked")
    @SafeVarargs
    public final <D> ComboTestHelper<X> withDimension(String name, DimensionSetter<X, D> setter, D... dims) {
        dimensionInfos.add(new DimensionInfo<>(name, dims, setter));
        return this;
    }

    /**
     * Adds a new array dimension to this combo test, with a given base name. This allows to specify
     * multiple dimensions at once; the names of the underlying dimensions will be generated from the
     * base name, using standard array bracket notation - i.e. "DIM[0]", "DIM[1]", etc.
     */
    @SafeVarargs
    public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, D... dims) {
        return withArrayDimension(name, null, size, dims);
    }

    /**
     * Adds a new array dimension to this combo test, with a given base name, an array of values and a
     * coresponding array setter to be called in order to set the dimension value on the combo test
     * instance (before test execution). This allows to specify multiple dimensions at once; the names
     * of the underlying dimensions will be generated from the base name, using standard array bracket
     * notation - i.e. "DIM[0]", "DIM[1]", etc.
     */
    @SafeVarargs
    public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, D... dims) {
        for (int i = 0 ; i < size ; i++) {
            dimensionInfos.add(new ArrayDimensionInfo<>(name, dims, i, setter));
        }
        return this;
    }

    /**
     * Returns the stat object associated with this combo test.
     */
    public Info info() {
        return info;
    }

    /**
     * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
     * execute a new test instance (built using given supplier) for each such combination.
     */
    public void run(Supplier<X> instanceBuilder) {
        run(instanceBuilder, null);
    }

    /**
     * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
     * execute a new test instance (built using given supplier) for each such combination. Before
     * executing the test instance entry point, the supplied initialization method is called on
     * the test instance; this is useful for ad-hoc test instance initialization once all the dimension
     * values have been set.
     */
    public void run(Supplier<X> instanceBuilder, Consumer<X> initAction) {
        runInternal(0, new Stack<>(), instanceBuilder, Optional.ofNullable(initAction));
        end();
    }

    /**
     * Generate combinatorial explosion of all dimension values and create a new test instance
     * for each combination.
     */
    @SuppressWarnings({"unchecked", "rawtypes"})
    private void runInternal(int index, Stack<DimensionBinding<?>> bindings, Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction) {
        if (index == dimensionInfos.size()) {
            runCombo(instanceBuilder, initAction, bindings);
        } else {
            DimensionInfo<?> dinfo = dimensionInfos.get(index);
            for (Object d : dinfo.dims) {
                bindings.push(new DimensionBinding(d, dinfo));
                runInternal(index + 1, bindings, instanceBuilder, initAction);
                bindings.pop();
            }
        }
    }

    /**
     * Run a new test instance using supplied dimension bindings. All required setters and initialization
     * method are executed before calling the instance main entry point. Also checks if the instance
     * is compatible with the specified test filters; if not, the test is simply skipped.
     */
    @SuppressWarnings("unchecked")
    private void runCombo(Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction, List<DimensionBinding<?>> bindings) {
        X x = instanceBuilder.get();
        for (DimensionBinding<?> binding : bindings) {
            binding.init(x);
        }
        initAction.ifPresent(action -> action.accept(x));
        info.comboCount++;
        if (!optFilter.isPresent() || optFilter.get().test(x)) {
            x.run(new Env(bindings));
            if (failMode.shouldStop(ignoreMode, info)) {
                end();
            }
        } else {
            info.skippedCount++;
        }
    }

    /**
     * This method is executed upon combo test completion (either normal or erroneous). Closes down
     * all pending resources and dumps useful stats info.
     */
    private void end() {
        try {
            fm.close();
            if (info.hasFailures()) {
                throw new AssertionError("Failure when executing combo:" + info.lastFailure.orElse(""));
            } else if (info.hasErrors()) {
                throw new AssertionError("Unexpected exception while executing combo", info.lastError.get());
            }
        } catch (IOException ex) {
            throw new AssertionError("Failure when closing down shared file manager; ", ex);
        } finally {
            info.dump();
        }
    }

    /**
     * Functional interface for specifying combo test instance setters.
     */
    public interface DimensionSetter<X extends ComboInstance<X>, D> {
        void set(X x, D d);
    }

    /**
     * Functional interface for specifying combo test instance array setters. The setter method
     * receives an extra argument for the index of the array element to be set.
     */
    public interface ArrayDimensionSetter<X extends ComboInstance<X>, D> {
        void set(X x, D d, int index);
    }

    /**
     * Dimension descriptor; each dimension has a name, an array of value and an optional setter
     * to be called on the associated combo test instance.
     */
    class DimensionInfo<D> {
        String name;
        D[] dims;
        boolean isParameter;
        Optional<DimensionSetter<X, D>> optSetter;

        DimensionInfo(String name, D[] dims, DimensionSetter<X, D> setter) {
            this.name = name;
            this.dims = dims;
            this.optSetter = Optional.ofNullable(setter);
            this.isParameter = dims[0] instanceof ComboParameter;
        }
    }

    /**
     * Array dimension descriptor. The dimension name is derived from a base name and an index using
     * standard bracket notation; ; the setter accepts an additional 'index' argument to point
     * to the array element to be initialized.
     */
    class ArrayDimensionInfo<D> extends DimensionInfo<D> {
        public ArrayDimensionInfo(String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {
            super(String.format("%s[%d]", name, index), dims,
                    setter != null ? (x, d) -> setter.set(x, d, index) : null);
        }
    }

    /**
     * Failure policies for a combo test run.
     */
    public enum FailMode {
        /** Combo test fails when first failure is detected. */
        FAIL_FAST,
        /** Combo test fails after all instances have been executed. */
        FAIL_AFTER;

        boolean shouldStop(IgnoreMode ignoreMode, Info info) {
            switch (this) {
                case FAIL_FAST:
                    return !ignoreMode.canIgnore(info);
                default:
                    return false;
            }
        }
    }

    /**
     * Ignore policies for a combo test run.
     */
    public enum IgnoreMode {
        /** No error or failure is ignored. */
        IGNORE_NONE,
        /** Only errors are ignored. */
        IGNORE_ERRORS,
        /** Only failures are ignored. */
        IGNORE_FAILURES,
        /** Both errors and failures are ignored. */
        IGNORE_ALL;

        boolean canIgnore(Info info) {
            switch (this) {
                case IGNORE_ERRORS:
                    return info.failCount == 0;
                case IGNORE_FAILURES:
                    return info.errCount == 0;
                case IGNORE_ALL:
                    return true;
                default:
                    return info.failCount == 0 && info.errCount == 0;
            }
        }
    }

    /**
     * A dimension binding. This is essentially a pair of a dimension value and its corresponding
     * dimension info.
     */
    class DimensionBinding<D> {
        D d;
        DimensionInfo<D> info;

        DimensionBinding(D d, DimensionInfo<D> info) {
            this.d = d;
            this.info = info;
        }

        void init(X x) {
            info.optSetter.ifPresent(setter -> setter.set(x, d));
        }

        public String toString() {
            return String.format("(%s -> %s)", info.name, d);
        }
    }

    /**
     * This class is used to keep track of combo tests stats; info such as numbero of failures/errors,
     * number of times a context has been shared/dropped are all recorder here.
     */
    public static class Info {
        int failCount;
        int errCount;
        int passCount;
        int comboCount;
        int skippedCount;
        int ctxReusedCount;
        int ctxDroppedCount;
        Optional<String> lastFailure = Optional.empty();
        Optional<Throwable> lastError = Optional.empty();

        void dump() {
            System.err.println(String.format("%d total checks executed", comboCount));
            System.err.println(String.format("%d successes found", passCount));
            System.err.println(String.format("%d failures found", failCount));
            System.err.println(String.format("%d errors found", errCount));
            System.err.println(String.format("%d skips found", skippedCount));
            System.err.println(String.format("%d contexts shared", ctxReusedCount));
            System.err.println(String.format("%d contexts dropped", ctxDroppedCount));
        }

        public boolean hasFailures() {
            return failCount != 0;
        }

        public boolean hasErrors() {
            return errCount != 0;
        }
    }

    /**
     * THe execution environment for a given combo test instance. An environment contains the
     * bindings for all the dimensions, along with the combo parameter cache (this is non-empty
     * only if one or more dimensions are subclasses of the {@code ComboParameter} interface).
     */
    class Env {
        List<DimensionBinding<?>> bindings;
        Map<String, ComboParameter> parametersCache = new HashMap<>();

        @SuppressWarnings({"Unchecked", "rawtypes"})
        Env(List<DimensionBinding<?>> bindings) {
            this.bindings = bindings;
            for (DimensionBinding<?> binding : bindings) {
                if (binding.info.isParameter) {
                    parametersCache.put(binding.info.name, (ComboParameter)binding.d);
                };
            }
        }

        Info info() {
            return ComboTestHelper.this.info();
        }

        StandardJavaFileManager fileManager() {
            return fm;
        }

        JavaCompiler javaCompiler() {
            return comp;
        }

        ReusableContext context() {
            return context;
        }

        ReusableContext setContext(ReusableContext context) {
            return ComboTestHelper.this.context = context;
        }
    }
}