jetCheck: introduce ImperativeCommand API for easier testing of stateful systems

jetCheck/src/jetCheck/Generator.java

@@ -2,7 +2,10 @@ package jetCheck;
 
 import org.jetbrains.annotations.NotNull;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -26,13 +29,16 @@ public class Generator<T> {
   /**
    * Creates a generator from a custom function, that creates objects of the given type based on the data from {@link DataStructure}.
    * The generator may call {@link DataStructure#drawInt} methods directly (and interpret those ints in any way it wishes),
-   * or invoke other generators using {@link DataStructure#generate(Generator)}.<p/>
+   * or (preferably) invoke other generators using {@link DataStructure#generate(Generator)}.<p/>
    * 
    * When a property is falsified, the DataStructure is attempted to be minimized, and the generator will be run on
    * ever "smaller" versions of it, this enables automatic minimization on all kinds of generated types.<p/>
    * 
-   * To ensure test reproducibility during re-run or minimization phase, generators must not have any internal state.
-   * Their result should only depend on the DataStructure.
+   * To ensure test reproducibility during re-run or minimization phase,
+   * the result of the generators should only depend on the DataStructure. Generators should not have any side effects
+   * or depend on the outside world. Generators may have internal mutable state accessible to other (nested) generators,
+   * but that's error-prone, difficult and computationally expensive to minimize. If you still think you need that,
+   * please see {@link ImperativeCommand} for potentially more convenient way of testing stateful systems.
    */
   @NotNull
   public static <T> Generator<T> from(@NotNull Function<DataStructure, T> function) {

jetCheck/src/jetCheck/ImperativeCommand.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+package jetCheck;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.function.Supplier;
+
+/**
+ * Represents an action with potential side effects, for single-threaded property-based testing of stateful systems.
+ * The engine executes a top-level command, which should prepare the system-under-test and run the sequence of nested (actual) commands
+ * that change the system, check the needed invariants, and log all activity to allow to reproduce failing scenarios.<p/>
+ * 
+ * A typical way to test with imperative commands looks like this:
+ * <pre>
+ *   ImperativeCommand.checkScenarios(() -> env -> {
+ *     System sys = setUpSystem();
+ *     try {
+ *       // run a random sequence of commands
+ *       env.performCommands(Generator.sampledFrom(new Command1(sys), new Command2(sys), ...))
+ *       assertPropertyHolds(sys); // optional; fail with an exception if some invariant is broken 
+ *     } finally {
+ *       tearDownSystem(); // if any resources should be freed
+ *     }
+ *   })
+ *   ...
+ *   class Command1 implements ImperativeCommand {
+ *     System sys;
+ *     Command1(System sys) {
+ *       this.sys = sys;
+ *     }
+ *     
+ *     public void performCommand(Environment env) {
+ *       env.logMessage("Perforing command1");
+ *       // do some actions on 'sys' using environment to generate random data (and log it) on the way, e.g.:
+ *       Item item = env.generateData(Generator.sampledFrom(sys.currentItems), "working on %s item");
+ *       modifyItemInTheSystem(sys, item); // may change the system and the items within it
+ *     }
+ *   }
+ *   ...
+ * </pre>
+ * 
+ * For more fine-grained control, you can use {@link #scenarios} to get a usual {@code PropertyChecker}. Then it's possible to
+ * customize, e.g. iteration count:
+ *   <pre>PropertyChecker.forAll(ImperativeCommand.scenarios(() -> env -> ...))
+ *   .withIterationCount(42)
+ *   .shouldHold(Scenario::ensureSuccessful)}</pre>
+ * 
+ * If any command fails with an exception, the property being checked is considered to be falsified and the test fails.
+ * In the error message, the causing exception is printed together with command log.
+ * Commands can and should log what they're doing using
+ * {@link Environment#logMessage(String)} and {@link Environment#generateValue(Generator, String)} so that you
+ * can restore the order of events that leads to the failure.<p/>
+ * 
+ * Top-level command should not have any side effects on the outside world, 
+ * otherwise proper test case separation and minimization will be impossible.
+ * The supplier that creates the top-level command should 
+ * return a "fresh" object each time, as it's invoked on each testing and minimization iterations. 
+ * Nested commands may have side effects, provided that those effects are fully contained within the system-under-test, 
+ * which is set up and disposed inside the top-level command on each iteration.<p/>
+ * 
+ * Test case minimization is complicated, when commands make random choices based on the current state of the system
+ * (which can change even after removing irrelevant commands during minimization).
+ * The engine tries to account for that heuristically. <b>Rule of thumb</b>: whenever your command generates an index into some list,
+ * or uses {@link Generator#sampledFrom} on system elements, try to ensure that these elements have some predictable ordering.
+ * It's ideal if each command effect on these elements is either adding or removing a contiguous range in that list. 
+ */
+public interface ImperativeCommand {
+  /** Perform the actual change on the system-under-test, using {@code env} to generate random data and log the progress */ 
+  void performCommand(@NotNull Environment env);
+
+  /** A helper object passed into {@link #performCommand} to allow for logging and ad hoc random data generation */
+  interface Environment {
+    
+    /** Add a log message. The whole execution log would be printed if the command fails */
+    void logMessage(@NotNull String message);
+    
+    /** Generate a pseudo-random value using the given generator.
+     * Optionally log a message, so that when a test fails, you'd know the value was generated.
+     * The message is a Java format string, so you can use it to include the generated value, e.g.
+     * {@code String s = generateValue(stringsOf(asciiLetters(), "Generated %s")}.<p/>
+     * If you don't want to generate message, or would like to show the generated value in a custom way, pass {@code null}.
+     * You can use {@link #logMessage} later to still leave a trace of this value generation in the log.
+     */
+    <T> T generateValue(@NotNull Generator<T> generator, @Nullable String logMessage);
+
+    /** Executes a sequence (of length with the given distribution) of random nested commands (produced by the given generator) */
+    void executeCommands(IntDistribution count, Generator<? extends ImperativeCommand> cmdGen);
+
+    /** Executes a non-empty sequence of random nested commands (produced by the given generator) */    
+    void executeCommands(Generator<? extends ImperativeCommand> cmdGen);
+  }
+
+  /** 
+   * @return a generator that runs the given command and returns a {@link Scenario} containing the execution log and result.
+   * @param command a supplier for a top-level command. This supplier should not have any side effects. 
+   * @see #checkScenarios
+   */
+  static Generator<Scenario> scenarios(@NotNull Supplier<ImperativeCommand> command) {
+    return Generator.from(data -> new ScenarioImpl(command.get(), data));
+  }
+
+  /**
+   * Performs a check that the scenarios generated by the given command are successful. Default {@link PropertyChecker} settings are used.
+   * @param command a supplier for a top-level command. This supplier should not have any side effects. 
+   */
+  static void checkScenarios(@NotNull Supplier<ImperativeCommand> command) {
+    PropertyChecker.forAll(scenarios(command)).shouldHold(Scenario::ensureSuccessful);
+  }
+
+  /**
+   * An analog of {@link PropertyChecker#rechecking} in the imperative command world.
+   * Performs a check that the scenario generated by the given command with given seed/sizeHint parameter is successful.
+   * @param command a supplier for a top-level command. This supplier should not have any side effects. 
+   */
+  static void checkScenario(long seed, int sizeHint, @NotNull Supplier<ImperativeCommand> command) {
+    PropertyChecker.forAll(scenarios(command)).rechecking(seed, sizeHint).shouldHold(Scenario::ensureSuccessful);
+  }
+
+}
+

jetCheck/src/jetCheck/Scenario.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+package jetCheck;
+
+/** Represents the execution history of an {@link ImperativeCommand} sequence */
+public interface Scenario {
+  /**
+   * This method is only useful when working with scenario generator explicitly ({@link ImperativeCommand#scenarios}) and
+   * asserting that the command execution was successful. For most cases, {@link ImperativeCommand#checkScenarios} is recommended instead.
+   * @return true if the command execution didn't result in an exception. Otherwise throws that exception.
+   * */
+  boolean ensureSuccessful();
+
+  /**
+   * Pretty-prints the log produced by the command execution.
+   * @see ImperativeCommand.Environment#logMessage(String)
+   * @see ImperativeCommand.Environment#generateValue(Generator, String) 
+   */
+  String toString();
+}

jetCheck/src/jetCheck/ScenarioImpl.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+package jetCheck;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+class ScenarioImpl implements Scenario {
+  private final List<Object> log = new ArrayList<>();
+  private Throwable failure;
+
+  ScenarioImpl(@NotNull ImperativeCommand cmd, @NotNull DataStructure data) {
+    try {
+      performCommand(cmd, data, log);
+    }
+    catch (Throwable e) {
+      addFailure(e);
+    }
+    if (failure instanceof CannotRestoreValue) {
+      throw (CannotRestoreValue)failure;
+    }
+  }
+
+  private void addFailure(Throwable e) {
+    if (failure == null) {
+      failure = e;
+    }
+  }
+
+  private void performCommand(ImperativeCommand command, DataStructure data, List<Object> log) {
+    command.performCommand(new ImperativeCommand.Environment() {
+      @Override
+      public void logMessage(@NotNull String message) {
+        log.add(message);
+      }
+
+      @Override
+      public <T> T generateValue(@NotNull Generator<T> generator, @Nullable String logMessage) {
+        T value = safeGenerate(data, generator);
+        if (logMessage != null) {
+          logMessage(String.format(logMessage, value));
+        }
+        return value;
+      }
+
+      @Override
+      public void executeCommands(IntDistribution count, Generator<? extends ImperativeCommand> cmdGen) {
+        data.generate(Generator.listsOf(count, innerCommands(cmdGen)));
+      }
+
+      @Override
+      public void executeCommands(Generator<? extends ImperativeCommand> cmdGen) {
+        data.generate(Generator.nonEmptyLists(innerCommands(cmdGen)));
+      }
+
+      @NotNull
+      private Generator<Object> innerCommands(Generator<? extends ImperativeCommand> cmdGen) {
+        return Generator.from(new Function<DataStructure, Object>() {
+          @Override
+          public Object apply(DataStructure cmdData) {
+            List<Object> localLog = new ArrayList<>();
+            log.add(localLog);
+            performCommand(safeGenerate(cmdData, cmdGen), cmdData, localLog);
+            return null;
+          }
+
+          @Override
+          public boolean equals(Object obj) {
+            return getClass() == obj.getClass(); // for recursive shrinking to work
+          }
+
+          @Override
+          public int hashCode() {
+            return getClass().hashCode();
+          }
+        });
+      }
+    });
+  }
+
+  private <T> T safeGenerate(DataStructure data, Generator<T> generator) {
+    try {
+      return data.generate(generator);
+    }
+    catch (CannotRestoreValue e) { //todo test for evil intermediate code hiding this exception, also CannotSatisfyCondition
+      addFailure(e);
+      throw e;
+    }
+  }
+
+
+  @Override
+  public boolean equals(Object o) {
+    return this == o || o instanceof ScenarioImpl && log.equals(((ScenarioImpl)o).log);
+  }
+
+  @Override
+  public int hashCode() {
+    return log.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    printLog(sb, "", log);
+    return "commands:" + (sb.length() == 0 ? "<none>" : sb.toString());
+  }
+  
+  private static void printLog(StringBuilder sb, String indent, List<Object> log) {
+    for (Object o : log) {
+      if (o instanceof String) {
+        sb.append("\n").append(indent).append(o);
+      } else {
+        //noinspection unchecked
+        printLog(sb, indent + "  ", (List)o);
+      }
+    }
+  }
+
+  @Override
+  public boolean ensureSuccessful() {
+    if (failure instanceof Error) throw (Error)failure;
+    if (failure instanceof RuntimeException) throw (RuntimeException)failure;
+    if (failure != null) throw new RuntimeException(failure);
+    return true;
+  }
+
+}

jetCheck/test/jetCheck/StatefulGeneratorTest.java

@@ -1,21 +1,25 @@
 // Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package jetCheck;
 
+import org.jetbrains.annotations.NotNull;
+
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static jetCheck.Generator.*;
+
 /**
  * @author peter
  */
 public class StatefulGeneratorTest extends PropertyCheckerTestCase {
 
   public void testShrinkingIntsWithDistributionsDependingOnListSize() {
-    Generator<List<InsertChar>> gen = Generator.from(data -> {
+    Generator<List<InsertChar>> gen = from(data -> {
       AtomicInteger modelLength = new AtomicInteger(0);
-      Generator<List<InsertChar>> cmds = Generator.listsOf(Generator.from(cmdData -> {
+      Generator<List<InsertChar>> cmds = listsOf(from(cmdData -> {
         int index = cmdData.drawInt(IntDistribution.uniform(0, modelLength.getAndIncrement()));
-        char c = cmdData.generate(Generator.asciiLetters());
+        char c = cmdData.generate(asciiLetters());
         return new InsertChar(c, index);
       }));
       return data.generate(cmds);
@@ -25,7 +29,48 @@ public class StatefulGeneratorTest extends PropertyCheckerTestCase {
                                                      64);
     assertEquals(minCmds.toString(), 2, minCmds.size());
   }
-  
+
+  public void testImperativeInsertDeleteCheckCommands() {
+    Scenario minHistory = checkFalsified(ImperativeCommand.scenarios(() -> env -> {
+      StringBuilder sb = new StringBuilder();
+      ImperativeCommand check = env1 -> {
+        env1.logMessage("check");
+        if (sb.indexOf("A") >= 0) throw new AssertionError();
+      };
+      env.executeCommands(recursive(rec -> {
+        ImperativeCommand group = env1 -> {
+          env1.logMessage("Group");
+          env1.executeCommands(rec);
+        };
+        return frequency(2, constant(group),
+                         3, sampledFrom(insertStringCmd(sb), deleteStringCmd(sb), check));
+      }));
+    }), Scenario::ensureSuccessful, 36).getMinimalCounterexample().getExampleValue();
+
+    assertEquals("commands:\n" +
+                 "  Group\n" +
+                 "    insert A at 0\n" +
+                 "    check", 
+                 minHistory.toString());
+  }
+
+  @NotNull
+  private static ImperativeCommand insertStringCmd(StringBuilder sb) {
+    return env -> {
+      int index = env.generateValue(integers(0, sb.length()), null);
+      String toInsert = env.generateValue(stringsOf(asciiLetters()), "insert %s at " + index);
+      sb.insert(index, toInsert);
+    };
+  }
+
+  @NotNull
+  private static ImperativeCommand deleteStringCmd(StringBuilder sb) {
+    return env -> {
+      int start = env.generateValue(integers(0, sb.length()), null);
+      int end = env.generateValue(integers(start, sb.length()), "deleting (" + start + ", %s)");
+      sb.delete(start, end);
+    };
+  }
 }
 
 class InsertChar {