jetCheck: smarter shrinking when item generation depends on previous items

For generators that generate sequences of commands with some internal model,
next commands might depend on the state of the model, modified by previous commands.

When previous commands are removed by shrinking, replaying generators would result
in a different models, and sampling from model elements would fail because
the model element counts have changed, and so should indices used for sampling.

Now we detect that we're given an int distribution that's different from the original,
and try to guess possible updated int values according to the new distribution.

jetCheck/src/jetCheck/BoundedIntDistribution.java

@@ -16,6 +16,14 @@ public class BoundedIntDistribution implements IntDistribution {
     this.producer = producer;
   }
 
+  int getMin() {
+    return min;
+  }
+
+  int getMax() {
+    return max;
+  }
+
   @Override
   public int generateInt(Random random) {
     int i = producer.applyAsInt(random);

jetCheck/src/jetCheck/IntCustomizer.java

@@ -0,0 +1,104 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package jetCheck;
+
+import org.jetbrains.annotations.Nullable;
+
+import java.util.*;
+
+/**
+ * @author peter
+ */
+
+interface IntCustomizer {
+  int suggestInt(IntData data, IntDistribution currentDistribution);
+
+  static int checkValidInt(IntData data, IntDistribution currentDistribution) {
+    int value = data.value;
+    if (!currentDistribution.isValidValue(value)) throw new CannotRestoreValue();
+    return value;
+  }
+
+}
+
+class CombinatorialIntCustomizer implements IntCustomizer {
+  private final LinkedHashMap<NodeId, Set<Integer>> valuesToTry;
+  private final Map<NodeId, Integer> currentCombination;
+  private final Map<NodeId, IntDistribution> changedDistributions = new HashMap<>();
+
+  CombinatorialIntCustomizer() {
+    this(new LinkedHashMap<>(), new HashMap<>());
+  }
+
+  private CombinatorialIntCustomizer(LinkedHashMap<NodeId, Set<Integer>> valuesToTry, Map<NodeId, Integer> currentCombination) {
+    this.valuesToTry = valuesToTry;
+    this.currentCombination = currentCombination;
+  }
+
+  public int suggestInt(IntData data, IntDistribution currentDistribution) {
+    if (data.distribution instanceof BoundedIntDistribution &&
+        currentDistribution instanceof BoundedIntDistribution &&
+        registerDifferentRange(data, (BoundedIntDistribution)currentDistribution, (BoundedIntDistribution)data.distribution)) {
+      return suggestCombinatorialVariant(data, currentDistribution);
+    }
+    return IntCustomizer.checkValidInt(data, currentDistribution);
+  }
+
+  private int suggestCombinatorialVariant(IntData data, IntDistribution currentDistribution) {
+    int value = currentCombination.getOrDefault(data.id, data.value);
+    if (currentDistribution.isValidValue(value)) {
+      currentCombination.putIfAbsent(data.id, value);
+      return value;
+    }
+
+    throw new CannotRestoreValue();
+  }
+
+  private boolean registerDifferentRange(IntData data, BoundedIntDistribution current, BoundedIntDistribution original) {
+    int oMax = original.getMax();
+    int cMax = current.getMax();
+    int oMin = original.getMin();
+    int cMin = current.getMin();
+    if (oMax != cMax || oMin != cMin) {
+      changedDistributions.put(data.id, current);
+
+      addValueToTry(data, current, data.value);
+      addValueToTry(data, current, (data.value - oMin) + cMin);
+      addValueToTry(data, current, cMax - (oMax - data.value));
+      return true;
+    }
+    return false;
+  }
+
+  private void addValueToTry(IntData data, IntDistribution current, int value) {
+    if (current.isValidValue(value)) {
+      valuesToTry.computeIfAbsent(data.id, __1 -> new LinkedHashSet<>()).add(value);
+    }
+  }
+
+  @Nullable
+  CombinatorialIntCustomizer nextAttempt() {
+    Map<NodeId, Integer> nextCombination = new HashMap<>(currentCombination);
+    for (Map.Entry<NodeId, Set<Integer>> entry : valuesToTry.entrySet()) {
+      List<Integer> possibleValues = new ArrayList<>(entry.getValue());
+      Integer usedValue = currentCombination.get(entry.getKey());
+      int index = possibleValues.indexOf(usedValue);
+      if (index < possibleValues.size() - 1) {
+        // found a position which can be incremented by 1
+        nextCombination.put(entry.getKey(), possibleValues.get(index + 1));
+        return new CombinatorialIntCustomizer(valuesToTry, nextCombination);
+      }
+      // digit overflow in this position, so zero it and try incrementing the next one
+      nextCombination.put(entry.getKey(), possibleValues.get(0));
+    }
+    return null;
+  }
+
+  StructureNode writeChanges(StructureNode node) {
+    StructureNode result = node;
+    for (Map.Entry<NodeId, IntDistribution> entry : changedDistributions.entrySet()) {
+      NodeId id = entry.getKey();
+      result = result.replace(id, new IntData(id, currentCombination.get(id), entry.getValue()));
+    }
+    return result;
+  }
+}

jetCheck/src/jetCheck/Iteration.java

@@ -75,7 +75,7 @@ class Iteration<T> {
     if (example != null) {
       session.notifier.counterExampleFound(this);
       PropertyFailureImpl<T> failure = new PropertyFailureImpl<>(example, this);
-      throw new PropertyFalsified(failure, () -> new ReplayDataStructure(failure.getMinimalCounterexample().data, sizeHint));
+      throw new PropertyFalsified(failure, () -> new ReplayDataStructure(failure.getMinimalCounterexample().data, sizeHint, IntCustomizer::checkValidInt));
     }
 
     if (iterationNumber >= session.iterationCount) {

jetCheck/src/jetCheck/PropertyFailureImpl.java

@@ -84,22 +84,25 @@ class PropertyFailureImpl<T> implements PropertyFailure<T> {
         StructureNode node = minimized.data.replace(replacedId, replacement);
         if (!iteration.session.generatedHashes.add(node.hashCode())) return false;
 
-        iteration.session.notifier.shrinkAttempt(PropertyFailureImpl.this, iteration);
+        CombinatorialIntCustomizer customizer = new CombinatorialIntCustomizer();
+        while (customizer != null) {
+          try {
+            iteration.session.notifier.shrinkAttempt(PropertyFailureImpl.this, iteration);
 
-        try {
-          T value = iteration.session.generator.getGeneratorFunction().apply(new ReplayDataStructure(node, iteration.sizeHint));
-          totalSteps++;
-          CounterExampleImpl<T> example = CounterExampleImpl.checkProperty(iteration.session.property, value, node);
-          if (example != null) {
-            minimized = example;
-            successfulSteps++;
-            return true;
+            T value = iteration.session.generator.getGeneratorFunction().apply(new ReplayDataStructure(node, iteration.sizeHint, customizer));
+            totalSteps++;
+            CounterExampleImpl<T> example = CounterExampleImpl.checkProperty(iteration.session.property, value, customizer.writeChanges(node));
+            if (example != null) {
+              minimized = example;
+              successfulSteps++;
+              return true;
+            }
           }
-          return false;
-        }
-        catch (CannotRestoreValue e) {
-          return false;
+          catch (CannotRestoreValue ignored) {
+          }
+          customizer = customizer.nextAttempt();
         }
+        return false;
       }
     });
   }

jetCheck/src/jetCheck/ReplayDataStructure.java

@@ -7,17 +7,17 @@ import java.util.function.Predicate;
 
 class ReplayDataStructure extends AbstractDataStructure {
   private final Iterator<StructureElement> iterator;
+  private final IntCustomizer customizer;
 
-  public ReplayDataStructure(StructureNode node, int sizeHint) {
+  public ReplayDataStructure(StructureNode node, int sizeHint, IntCustomizer customizer) {
     super(node, sizeHint);
     this.iterator = node.childrenIterator();
+    this.customizer = customizer;
   }
 
   @Override
   public int drawInt(@NotNull IntDistribution distribution) {
-    int value = nextChild(IntData.class).value;
-    if (!distribution.isValidValue(value)) throw new CannotRestoreValue();
-    return value;
+    return customizer.suggestInt(nextChild(IntData.class), distribution);
   }
 
   @NotNull
@@ -32,7 +32,7 @@ class ReplayDataStructure extends AbstractDataStructure {
   @NotNull
   @Override
   public DataStructure subStructure() {
-    return new ReplayDataStructure(nextChild(StructureNode.class), childSizeHint());
+    return new ReplayDataStructure(nextChild(StructureNode.class), childSizeHint(), customizer);
   }
 
   @Override

jetCheck/src/jetCheck/StructureNode.java

@@ -7,7 +7,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 /**
@@ -84,13 +83,14 @@ class StructureNode extends StructureElement {
   }
 
   private static StructureNode shrinkList(ShrinkContext context, StructureNode node) {
-    int start = 1;
-    int length = 1;
     int limit = node.children.size();
     while (limit > 0) {
+      int start = 1;
+      int length = 1;
       int lastSuccessfulRemove = -1;
       while (start < limit && start < node.children.size()) {
-        if (context.tryReplacement(node.id, node.removeRange(node.children, start, length))) {
+        // remove last items from the end first to decrease the number of variants in CombinatorialIntCustomizer
+        if (context.tryReplacement(node.id, node.removeRange(node.children, node.children.size() - start - length + 1, length))) {
           node = (StructureNode)Objects.requireNonNull(context.getCurrentMinimalRoot().findChildById(node.id));
           length = Math.min(length * 2, node.children.size() - start);
           lastSuccessfulRemove = start;

jetCheck/test/jetCheck/GeneratorTest.java

@@ -13,7 +13,7 @@ import static jetCheck.Generator.*;
  * @author peter
  */
 public class GeneratorTest extends PropertyCheckerTestCase {
-  
+
   public void testMod() {
     checkFalsified(integers(),
                    i -> i % 12 != 0,
@@ -27,16 +27,15 @@ public class GeneratorTest extends PropertyCheckerTestCase {
   }
 
   public void testListContainsDivisible() {
-    checkFalsified(nonEmptyLists(integers()),
-                   l -> l.stream().allMatch(i -> i % 10 != 0),
-                   3);
+    checkGeneratesExample(nonEmptyLists(integers()),
+                          l -> l.stream().anyMatch(i -> i % 10 == 0),
+                          4);
   }
 
   public void testStringContains() {
-    PropertyFailure<String> failure = checkFalsified(stringsOf(asciiPrintableChars()),
-                                               s -> !s.contains("a"),
-                                               17);
-    assertEquals("a", failure.getMinimalCounterexample().getExampleValue());
+    assertEquals("a", checkGeneratesExample(stringsOf(asciiPrintableChars()),
+                                            s -> s.contains("a"),
+                                            16));
   }
 
   public void testLetterStringContains() {
@@ -44,7 +43,7 @@ public class GeneratorTest extends PropertyCheckerTestCase {
                    s -> !s.contains("a"),
                    3);
   }
-  
+
   public void testIsSorted() {
     PropertyFailure<List<Integer>> failure = checkFalsified(nonEmptyLists(integers()),
                                                             l -> l.stream().sorted().collect(Collectors.toList()).equals(l),
@@ -61,7 +60,7 @@ public class GeneratorTest extends PropertyCheckerTestCase {
   public void testSortedDoublesNonDescending() {
     PropertyFailure<List<Double>> failure = checkFalsified(listsOf(doubles()),
                                                            l -> isSorted(l.stream().sorted().collect(Collectors.toList())),
-                                                           28);
+                                                           26);
     assertEquals(2, failure.getMinimalCounterexample().getExampleValue().size());
   }
 
@@ -110,23 +109,23 @@ public class GeneratorTest extends PropertyCheckerTestCase {
   public void testAsciiIdentifier() {
     PropertyChecker.forAll(asciiIdentifiers())
       .shouldHold(s -> Character.isJavaIdentifierStart(s.charAt(0)) && s.chars().allMatch(Character::isJavaIdentifierPart));
-    checkFalsified(asciiIdentifiers(),
-                   s -> !s.contains("_"),
-                   3);
+    checkGeneratesExample(asciiIdentifiers(),
+                          s -> s.contains("_"),
+                          2);
   }
 
   public void testBoolean() {
-    PropertyFailure<List<Boolean>> failure = checkFalsified(listsOf(booleans()),
-                                                            l -> !l.contains(true) || !l.contains(false),
-                                                            2);
-    assertEquals(2, failure.getMinimalCounterexample().getExampleValue().size());
+    List<Boolean> list = checkGeneratesExample(listsOf(booleans()),
+                                               l -> l.contains(true) && l.contains(false),
+                                               3);
+    assertEquals(2, list.size());
   }
 
   public void testShrinkingNonEmptyList() {
-    PropertyFailure<List<Integer>> failure = checkFalsified(nonEmptyLists(integers(0, 100)),
-                                                            l -> !l.contains(42),
-                                                            4);
-    assertEquals(1, failure.getMinimalCounterexample().getExampleValue().size());
+    List<Integer> list = checkGeneratesExample(nonEmptyLists(integers(0, 100)),
+                                               l -> l.contains(42),
+                                               5);
+    assertEquals(1, list.size());
   }
 
   public void testRecheckWithGivenSeeds() {
@@ -151,17 +150,17 @@ public class GeneratorTest extends PropertyCheckerTestCase {
                      String s = l.toString();
                      return !"abcdefghijklmnopqrstuvwxyz()[]#!".chars().allMatch(c -> s.indexOf((char)c) >= 0);
                    },
-                   153);
+                   244);
   }
 
   public void testSameFrequency() {
-    checkFalsified(listsOf(frequency(1, constant(1), 1, constant(2))), 
-                   l -> !l.contains(1) || !l.contains(2), 
+    checkFalsified(listsOf(frequency(1, constant(1), 1, constant(2))),
+                   l -> !l.contains(1) || !l.contains(2),
                    2);
-    
-    checkFalsified(listsOf(frequency(1, constant(1), 1, constant(2)).with(1, constant(3))), 
-                   l -> !l.contains(1) || !l.contains(2) || !l.contains(3), 
-                   10);
+
+    checkFalsified(listsOf(frequency(1, constant(1), 1, constant(2)).with(1, constant(3))),
+                   l -> !l.contains(1) || !l.contains(2) || !l.contains(3),
+                   17);
   }
 
 }

jetCheck/test/jetCheck/PropertyCheckerTestCase.java

@@ -19,6 +19,10 @@ abstract class PropertyCheckerTestCase extends TestCase {
     }
   }
 
+  protected <T> T checkGeneratesExample(Generator<T> generator, Predicate<T> predicate, int minimizationSteps) {
+    return checkFalsified(generator, predicate.negate(), minimizationSteps).getMinimalCounterexample().getExampleValue();
+  }
+
   protected <T> PropertyFailure<T> checkFalsified(Generator<T> generator, Predicate<T> predicate, int minimizationSteps) {
     PropertyFalsified e = checkFails(forAllStable(generator), predicate);
     //noinspection unchecked

jetCheck/test/jetCheck/StatefulGeneratorTest.java

@@ -0,0 +1,67 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package jetCheck;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @author peter
+ */
+public class StatefulGeneratorTest extends PropertyCheckerTestCase {
+
+  public void testShrinkingIntsWithDistributionsDependingOnListSize() {
+    Generator<List<InsertChar>> gen = Generator.from(data -> {
+      AtomicInteger modelLength = new AtomicInteger(0);
+      Generator<List<InsertChar>> cmds = Generator.listsOf(Generator.from(cmdData -> {
+        int index = cmdData.drawInt(IntDistribution.uniform(0, modelLength.getAndIncrement()));
+        char c = Generator.asciiLetters().generateValue(cmdData);
+        return new InsertChar(c, index);
+      }));
+      return cmds.generateValue(data);
+    });
+    List<InsertChar> minCmds = checkGeneratesExample(gen,
+                                                   cmds -> InsertChar.performOperations(cmds).contains("ab"),
+                                                     31);
+    assertEquals(minCmds.toString(), 2, minCmds.size());
+  }
+  
+}
+
+class InsertChar {
+  private final char c;
+  private final int index;
+
+  InsertChar(char c, int index) {
+    this.c = c;
+    this.index = index;
+  }
+
+  public void performOperation(StringBuilder sb) {
+    sb.insert(index, c);
+  }
+
+  @Override
+  public String toString() {
+    return "insert " + c + " at " + index;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof InsertChar)) return false;
+    InsertChar aChar = (InsertChar)o;
+    return c == aChar.c && index == aChar.index;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(c, index);
+  }
+
+  static String performOperations(List<InsertChar> cmds) {
+    StringBuilder sb = new StringBuilder();
+    cmds.forEach(cmd -> cmd.performOperation(sb));
+    return sb.toString();
+  }
+}