diff --git a/jetCheck/src/jetCheck/Generator.java b/jetCheck/src/jetCheck/Generator.java index 941cca9dbb6e..c3c0337754da 100644 --- a/jetCheck/src/jetCheck/Generator.java +++ b/jetCheck/src/jetCheck/Generator.java @@ -2,7 +2,10 @@ package jetCheck; import org.jetbrains.annotations.NotNull; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Function; @@ -26,13 +29,16 @@ public class Generator { /** * Creates a generator from a custom function, that creates objects of the given type based on the data from {@link DataStructure}. * The generator may call {@link DataStructure#drawInt} methods directly (and interpret those ints in any way it wishes), - * or invoke other generators using {@link DataStructure#generate(Generator)}.

+ * or (preferably) invoke other generators using {@link DataStructure#generate(Generator)}.

* * When a property is falsified, the DataStructure is attempted to be minimized, and the generator will be run on * ever "smaller" versions of it, this enables automatic minimization on all kinds of generated types.

* - * To ensure test reproducibility during re-run or minimization phase, generators must not have any internal state. - * Their result should only depend on the DataStructure. + * To ensure test reproducibility during re-run or minimization phase, + * the result of the generators should only depend on the DataStructure. Generators should not have any side effects + * or depend on the outside world. Generators may have internal mutable state accessible to other (nested) generators, + * but that's error-prone, difficult and computationally expensive to minimize. If you still think you need that, + * please see {@link ImperativeCommand} for potentially more convenient way of testing stateful systems. */ @NotNull public static Generator from(@NotNull Function function) { diff --git a/jetCheck/src/jetCheck/ImperativeCommand.java b/jetCheck/src/jetCheck/ImperativeCommand.java new file mode 100644 index 000000000000..6c6655db13ca --- /dev/null +++ b/jetCheck/src/jetCheck/ImperativeCommand.java @@ -0,0 +1,123 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. + */ +package jetCheck; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.function.Supplier; + +/** + * Represents an action with potential side effects, for single-threaded property-based testing of stateful systems. + * The engine executes a top-level command, which should prepare the system-under-test and run the sequence of nested (actual) commands + * that change the system, check the needed invariants, and log all activity to allow to reproduce failing scenarios.

+ * + * A typical way to test with imperative commands looks like this: + * 

+ *   ImperativeCommand.checkScenarios(() -> env -> {
+ *     System sys = setUpSystem();
+ *     try {
+ *       // run a random sequence of commands
+ *       env.performCommands(Generator.sampledFrom(new Command1(sys), new Command2(sys), ...))
+ *       assertPropertyHolds(sys); // optional; fail with an exception if some invariant is broken 
+ *     } finally {
+ *       tearDownSystem(); // if any resources should be freed
+ *     }
+ *   })
+ *   ...
+ *   class Command1 implements ImperativeCommand {
+ *     System sys;
+ *     Command1(System sys) {
+ *       this.sys = sys;
+ *     }
+ *     
+ *     public void performCommand(Environment env) {
+ *       env.logMessage("Perforing command1");
+ *       // do some actions on 'sys' using environment to generate random data (and log it) on the way, e.g.:
+ *       Item item = env.generateData(Generator.sampledFrom(sys.currentItems), "working on %s item");
+ *       modifyItemInTheSystem(sys, item); // may change the system and the items within it
+ *     }
+ *   }
+ *   ...
+ *
+ * + * For more fine-grained control, you can use {@link #scenarios} to get a usual {@code PropertyChecker}. Then it's possible to + * customize, e.g. iteration count: + * 
PropertyChecker.forAll(ImperativeCommand.scenarios(() -> env -> ...))
+ *   .withIterationCount(42)
+ *   .shouldHold(Scenario::ensureSuccessful)}
+ * + * If any command fails with an exception, the property being checked is considered to be falsified and the test fails. + * In the error message, the causing exception is printed together with command log. + * Commands can and should log what they're doing using + * {@link Environment#logMessage(String)} and {@link Environment#generateValue(Generator, String)} so that you + * can restore the order of events that leads to the failure.

+ * + * Top-level command should not have any side effects on the outside world, + * otherwise proper test case separation and minimization will be impossible. + * The supplier that creates the top-level command should + * return a "fresh" object each time, as it's invoked on each testing and minimization iterations. + * Nested commands may have side effects, provided that those effects are fully contained within the system-under-test, + * which is set up and disposed inside the top-level command on each iteration.

+ * + * Test case minimization is complicated, when commands make random choices based on the current state of the system + * (which can change even after removing irrelevant commands during minimization). + * The engine tries to account for that heuristically. Rule of thumb: whenever your command generates an index into some list, + * or uses {@link Generator#sampledFrom} on system elements, try to ensure that these elements have some predictable ordering. + * It's ideal if each command effect on these elements is either adding or removing a contiguous range in that list. + */ +public interface ImperativeCommand { + /** Perform the actual change on the system-under-test, using {@code env} to generate random data and log the progress */ + void performCommand(@NotNull Environment env); + + /** A helper object passed into {@link #performCommand} to allow for logging and ad hoc random data generation */ + interface Environment { + + /** Add a log message. The whole execution log would be printed if the command fails */ + void logMessage(@NotNull String message); + + /** Generate a pseudo-random value using the given generator. + * Optionally log a message, so that when a test fails, you'd know the value was generated. + * The message is a Java format string, so you can use it to include the generated value, e.g. + * {@code String s = generateValue(stringsOf(asciiLetters(), "Generated %s")}.

+ * If you don't want to generate message, or would like to show the generated value in a custom way, pass {@code null}. + * You can use {@link #logMessage} later to still leave a trace of this value generation in the log. + */ + T generateValue(@NotNull Generator generator, @Nullable String logMessage); + + /** Executes a sequence (of length with the given distribution) of random nested commands (produced by the given generator) */ + void executeCommands(IntDistribution count, Generator cmdGen); + + /** Executes a non-empty sequence of random nested commands (produced by the given generator) */ + void executeCommands(Generator cmdGen); + } + + /** + * @return a generator that runs the given command and returns a {@link Scenario} containing the execution log and result. + * @param command a supplier for a top-level command. This supplier should not have any side effects. + * @see #checkScenarios + */ + static Generator scenarios(@NotNull Supplier command) { + return Generator.from(data -> new ScenarioImpl(command.get(), data)); + } + + /** + * Performs a check that the scenarios generated by the given command are successful. Default {@link PropertyChecker} settings are used. + * @param command a supplier for a top-level command. This supplier should not have any side effects. + */ + static void checkScenarios(@NotNull Supplier command) { + PropertyChecker.forAll(scenarios(command)).shouldHold(Scenario::ensureSuccessful); + } + + /** + * An analog of {@link PropertyChecker#rechecking} in the imperative command world. + * Performs a check that the scenario generated by the given command with given seed/sizeHint parameter is successful. + * @param command a supplier for a top-level command. This supplier should not have any side effects. + */ + static void checkScenario(long seed, int sizeHint, @NotNull Supplier command) { + PropertyChecker.forAll(scenarios(command)).rechecking(seed, sizeHint).shouldHold(Scenario::ensureSuccessful); + } + +} + diff --git a/jetCheck/src/jetCheck/Scenario.java b/jetCheck/src/jetCheck/Scenario.java new file mode 100644 index 000000000000..80b09f50c06e --- /dev/null +++ b/jetCheck/src/jetCheck/Scenario.java @@ -0,0 +1,21 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. + */ +package jetCheck; + +/** Represents the execution history of an {@link ImperativeCommand} sequence */ +public interface Scenario { + /** + * This method is only useful when working with scenario generator explicitly ({@link ImperativeCommand#scenarios}) and + * asserting that the command execution was successful. For most cases, {@link ImperativeCommand#checkScenarios} is recommended instead. + * @return true if the command execution didn't result in an exception. Otherwise throws that exception. + * */ + boolean ensureSuccessful(); + + /** + * Pretty-prints the log produced by the command execution. + * @see ImperativeCommand.Environment#logMessage(String) + * @see ImperativeCommand.Environment#generateValue(Generator, String) + */ + String toString(); +} diff --git a/jetCheck/src/jetCheck/ScenarioImpl.java b/jetCheck/src/jetCheck/ScenarioImpl.java new file mode 100644 index 000000000000..f37c6c17d0f5 --- /dev/null +++ b/jetCheck/src/jetCheck/ScenarioImpl.java @@ -0,0 +1,133 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. + */ +package jetCheck; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +class ScenarioImpl implements Scenario { + private final List log = new ArrayList<>(); + private Throwable failure; + + ScenarioImpl(@NotNull ImperativeCommand cmd, @NotNull DataStructure data) { + try { + performCommand(cmd, data, log); + } + catch (Throwable e) { + addFailure(e); + } + if (failure instanceof CannotRestoreValue) { + throw (CannotRestoreValue)failure; + } + } + + private void addFailure(Throwable e) { + if (failure == null) { + failure = e; + } + } + + private void performCommand(ImperativeCommand command, DataStructure data, List log) { + command.performCommand(new ImperativeCommand.Environment() { + @Override + public void logMessage(@NotNull String message) { + log.add(message); + } + + @Override + public T generateValue(@NotNull Generator generator, @Nullable String logMessage) { + T value = safeGenerate(data, generator); + if (logMessage != null) { + logMessage(String.format(logMessage, value)); + } + return value; + } + + @Override + public void executeCommands(IntDistribution count, Generator cmdGen) { + data.generate(Generator.listsOf(count, innerCommands(cmdGen))); + } + + @Override + public void executeCommands(Generator cmdGen) { + data.generate(Generator.nonEmptyLists(innerCommands(cmdGen))); + } + + @NotNull + private Generator innerCommands(Generator cmdGen) { + return Generator.from(new Function() { + @Override + public Object apply(DataStructure cmdData) { + List localLog = new ArrayList<>(); + log.add(localLog); + performCommand(safeGenerate(cmdData, cmdGen), cmdData, localLog); + return null; + } + + @Override + public boolean equals(Object obj) { + return getClass() == obj.getClass(); // for recursive shrinking to work + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + }); + } + }); + } + + private T safeGenerate(DataStructure data, Generator generator) { + try { + return data.generate(generator); + } + catch (CannotRestoreValue e) { //todo test for evil intermediate code hiding this exception, also CannotSatisfyCondition + addFailure(e); + throw e; + } + } + + + @Override + public boolean equals(Object o) { + return this == o || o instanceof ScenarioImpl && log.equals(((ScenarioImpl)o).log); + } + + @Override + public int hashCode() { + return log.hashCode(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + printLog(sb, "", log); + return "commands:" + (sb.length() == 0 ? "" : sb.toString()); + } + + private static void printLog(StringBuilder sb, String indent, List log) { + for (Object o : log) { + if (o instanceof String) { + sb.append("\n").append(indent).append(o); + } else { + //noinspection unchecked + printLog(sb, indent + " ", (List)o); + } + } + } + + @Override + public boolean ensureSuccessful() { + if (failure instanceof Error) throw (Error)failure; + if (failure instanceof RuntimeException) throw (RuntimeException)failure; + if (failure != null) throw new RuntimeException(failure); + return true; + } + +} diff --git a/jetCheck/test/jetCheck/StatefulGeneratorTest.java b/jetCheck/test/jetCheck/StatefulGeneratorTest.java index 947a3f73ccb7..e5a05ff7ca55 100644 --- a/jetCheck/test/jetCheck/StatefulGeneratorTest.java +++ b/jetCheck/test/jetCheck/StatefulGeneratorTest.java @@ -1,21 +1,25 @@ // Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package jetCheck; +import org.jetbrains.annotations.NotNull; + import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import static jetCheck.Generator.*; + /** * @author peter */ public class StatefulGeneratorTest extends PropertyCheckerTestCase { public void testShrinkingIntsWithDistributionsDependingOnListSize() { - Generator> gen = Generator.from(data -> { + Generator> gen = from(data -> { AtomicInteger modelLength = new AtomicInteger(0); - Generator> cmds = Generator.listsOf(Generator.from(cmdData -> { + Generator> cmds = listsOf(from(cmdData -> { int index = cmdData.drawInt(IntDistribution.uniform(0, modelLength.getAndIncrement())); - char c = cmdData.generate(Generator.asciiLetters()); + char c = cmdData.generate(asciiLetters()); return new InsertChar(c, index); })); return data.generate(cmds); @@ -25,7 +29,48 @@ public class StatefulGeneratorTest extends PropertyCheckerTestCase { 64); assertEquals(minCmds.toString(), 2, minCmds.size()); } - + + public void testImperativeInsertDeleteCheckCommands() { + Scenario minHistory = checkFalsified(ImperativeCommand.scenarios(() -> env -> { + StringBuilder sb = new StringBuilder(); + ImperativeCommand check = env1 -> { + env1.logMessage("check"); + if (sb.indexOf("A") >= 0) throw new AssertionError(); + }; + env.executeCommands(recursive(rec -> { + ImperativeCommand group = env1 -> { + env1.logMessage("Group"); + env1.executeCommands(rec); + }; + return frequency(2, constant(group), + 3, sampledFrom(insertStringCmd(sb), deleteStringCmd(sb), check)); + })); + }), Scenario::ensureSuccessful, 36).getMinimalCounterexample().getExampleValue(); + + assertEquals("commands:\n" + + " Group\n" + + " insert A at 0\n" + + " check", + minHistory.toString()); + } + + @NotNull + private static ImperativeCommand insertStringCmd(StringBuilder sb) { + return env -> { + int index = env.generateValue(integers(0, sb.length()), null); + String toInsert = env.generateValue(stringsOf(asciiLetters()), "insert %s at " + index); + sb.insert(index, toInsert); + }; + } + + @NotNull + private static ImperativeCommand deleteStringCmd(StringBuilder sb) { + return env -> { + int start = env.generateValue(integers(0, sb.length()), null); + int end = env.generateValue(integers(start, sb.length()), "deleting (" + start + ", %s)"); + sb.delete(start, end); + }; + } } class InsertChar {