MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

slowCheck: add some basic flakiness detection

Browse Source
This commit is contained in:
peter
2017-06-29 14:27:36 +02:00
parent 89c0cd140e
commit 743afced9f
5 changed files with 143 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
slowCheck/src/slowCheck/Generator.java
+5 -1
View File
@@ -34,7 +34,11 @@ public final class Generator<T> {
}
public <V> Generator<V> map(@NotNull Function<T,V> fun) {
return from(data -> fun.apply(generateUnstructured(data)));
return from(data -> fun.apply(myFunction.apply(data)));
}
public <V> Generator<V> flatMap(@NotNull Function<T,Generator<V>> fun) {
return from(data -> fun.apply(generateValue(data)).generateValue(data));
}
public Generator<T> noShrink() {
slowCheck/src/slowCheck/PropertyFalsified.java
+54 -6
View File
@@ -1,5 +1,9 @@
package slowCheck;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Supplier;
/**
@@ -7,19 +11,63 @@ import java.util.function.Supplier;
*/
@SuppressWarnings("ExceptionClassNameDoesntEndWithException")
public class PropertyFalsified extends RuntimeException {
static final String FAILURE_REASON_HAS_CHANGED_DURING_MINIMIZATION = "!!! FAILURE REASON HAS CHANGED DURING MINIMIZATION !!!";
private final long seed;
private final PropertyFailure<?> failure;
private final Supplier<DataStructure> data;
public PropertyFalsified(int seed, PropertyFailure<?> failure, Supplier<DataStructure> data) {
super("Falsified on " + failure.getMinimalCounterexample().getExampleValue() + "\n" +
(failure.getTotalMinimizationStepCount() > 0
? "Minimized in " + failure.getTotalMinimizationStepCount() + " steps\n"
: "") +
"Seed=" + seed, failure.getMinimalCounterexample().getExceptionCause());
PropertyFalsified(long seed, PropertyFailure<?> failure, Supplier<DataStructure> data) {
super(failure.getMinimalCounterexample().getExceptionCause());
this.seed = seed;
this.failure = failure;
this.data = data;
}
@Override
public String getMessage() {
int minimizationStepCount = failure.getTotalMinimizationStepCount();
String msg = "Falsified on " + failure.getMinimalCounterexample().getExampleValue() + "\n" +
(minimizationStepCount > 0 ? "Minimized in " + minimizationStepCount + " steps\n" : "") +
"Seed=" + seed;
Throwable first = failure.getFirstCounterExample().getExceptionCause();
if (exceptionsDiffer(first, failure.getMinimalCounterexample().getExceptionCause())) {
msg += "\n " + FAILURE_REASON_HAS_CHANGED_DURING_MINIMIZATION;
if (first != null) {
msg += "\n Initial exception: " + printStackTrace(first) + "\n==========================\n";
} else {
msg += "\n Initially property was falsified without exceptions\n";
}
}
return msg;
}
private static boolean exceptionsDiffer(Throwable e1, Throwable e2) {
if (e1 == null && e2 == null) return false;
if ((e1 == null) != (e2 == null)) return true;
if (!e1.getClass().equals(e2.getClass())) return true;
return !getUserTrace(e1).equals(getUserTrace(e2));
}
private static List<String> getUserTrace(Throwable e) {
List<String> result = new ArrayList<>();
for (StackTraceElement element : e.getStackTrace()) {
String s = element.toString();
if (s.startsWith("slowCheck.CounterExampleImpl.checkProperty")) {
break;
}
result.add(s);
}
return result;
}
private static String printStackTrace(Throwable e) {
StringWriter stringWriter = new StringWriter();
PrintWriter writer = new PrintWriter(stringWriter);
e.printStackTrace(writer);
return stringWriter.getBuffer().toString();
}
public PropertyFailure<?> getFailure() {
return failure;
}
slowCheck/test/slowCheck/ExceptionTest.java
+51
View File
@@ -0,0 +1,51 @@
package slowCheck;
import static slowCheck.Generator.integers;
/**
* @author peter
*/
public class ExceptionTest extends PropertyCheckerTestCase {
public void testFailureReasonUnchanged() {
try {
PropertyChecker.forAll(ourTestSettings, integers(), i -> {
throw new AssertionError("fail");
});
fail();
}
catch (PropertyFalsified e) {
assertFalse(e.getMessage().contains(PropertyFalsified.FAILURE_REASON_HAS_CHANGED_DURING_MINIMIZATION));
}
}
public void testFailureReasonChangedExceptionClass() {
try {
PropertyChecker.forAll(ourTestSettings, integers(), i -> {
throw (i == 0 ? new RuntimeException("fail") : new IllegalArgumentException("fail"));
});
fail();
}
catch (PropertyFalsified e) {
assertTrue(e.getMessage().contains(PropertyFalsified.FAILURE_REASON_HAS_CHANGED_DURING_MINIMIZATION));
}
}
public void testFailureReasonChangedExceptionTrace() {
try {
PropertyChecker.forAll(ourTestSettings, integers(), i -> {
if (i == 0) {
throw new AssertionError("fail");
}
else {
throw new AssertionError("fail2");
}
});
fail();
}
catch (PropertyFalsified e) {
assertTrue(e.getMessage().contains(PropertyFalsified.FAILURE_REASON_HAS_CHANGED_DURING_MINIMIZATION));
}
}
}
slowCheck/test/slowCheck/GeneratorTest.java
+1 -29
View File
@@ -1,12 +1,9 @@
package slowCheck;
import junit.framework.TestCase;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static slowCheck.Generator.*;
@@ -14,8 +11,7 @@ import static slowCheck.Generator.*;
/**
* @author peter
*/
public class GeneratorTest extends TestCase {
private static final CheckerSettings ourTestSettings = CheckerSettings.DEFAULT_SETTINGS.withSeed(0);
public class GeneratorTest extends PropertyCheckerTestCase {
public void testMod() {
checkFalsified(integers(),
@@ -75,12 +71,6 @@ public class GeneratorTest extends TestCase {
return true;
}
public void testPropertyThrowsException() {
checkFalsified(integers(), p -> {
throw new AssertionError(p);
}, 1);
}
public void testSuchThat() {
PropertyChecker.forAll(integers().suchThat(i -> i < 0), i -> i < 0);
}
@@ -145,22 +135,4 @@ public class GeneratorTest extends TestCase {
assertEquals(1, failure.getMinimalCounterexample().getExampleValue().size());
}
private <T> PropertyFailure<T> checkFalsified(Generator<T> generator, Predicate<T> predicate, int minimizationSteps) {
try {
PropertyChecker.forAll(ourTestSettings, generator, predicate);
throw new AssertionError("Can't falsify " + getName());
}
catch (PropertyFalsified e) {
//noinspection unchecked
PropertyFailure<T> failure = (PropertyFailure<T>)e.getFailure();
System.out.println(" " + getName());
System.out.println("Value: " + e.getBreakingValue());
System.out.println("Data: " + e.getData());
assertEquals(minimizationSteps, failure.getTotalMinimizationStepCount());
assertEquals(e.getBreakingValue(), generator.generateUnstructured(e.getData()));
return failure;
}
}
}
slowCheck/test/slowCheck/PropertyCheckerTestCase.java
+32
View File
@@ -0,0 +1,32 @@
package slowCheck;
import junit.framework.TestCase;
import java.util.function.Predicate;
/**
* @author peter
*/
abstract class PropertyCheckerTestCase extends TestCase {
protected static final CheckerSettings ourTestSettings = CheckerSettings.DEFAULT_SETTINGS.withSeed(0);
protected <T> PropertyFailure<T> checkFalsified(Generator<T> generator, Predicate<T> predicate, int minimizationSteps) {
try {
PropertyChecker.forAll(ourTestSettings, generator, predicate);
throw new AssertionError("Can't falsify " + getName());
}
catch (PropertyFalsified e) {
//noinspection unchecked
PropertyFailure<T> failure = (PropertyFailure<T>)e.getFailure();
System.out.println(" " + getName());
System.out.println("Value: " + e.getBreakingValue());
System.out.println("Data: " + e.getData());
assertEquals(minimizationSteps, failure.getTotalMinimizationStepCount());
assertEquals(e.getBreakingValue(), generator.generateUnstructured(e.getData()));
return failure;
}
}
}