performance tests divided into CPU bound and IO bound

This commit is contained in:
Alexey Kudravtsev
2011-06-09 14:50:40 +04:00
parent 6dbc32828d
commit 3edddc0f85
13 changed files with 174 additions and 109 deletions
@@ -16,6 +16,7 @@ import com.intellij.psi.PsiDocumentManager;
import com.intellij.psi.impl.source.tree.injected.JavaConcatenationInjectorManager;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl;
import com.intellij.util.ThrowableRunnable;
import org.jetbrains.annotations.NonNls;
import java.util.ArrayList;
@@ -73,7 +74,7 @@ public class LightAdvHighlightingPerformanceTest extends LightDaemonAnalyzerTest
}
}
private List<HighlightInfo> doTest(final long maxMillis) throws Exception {
private List<HighlightInfo> doTest(final int maxMillis) throws Exception {
@NonNls String filePath = LightAdvHighlightingTest.BASE_PATH + "/" + getTestName(false) + ".java";
configureByFile(filePath);
@@ -82,15 +83,15 @@ public class LightAdvHighlightingPerformanceTest extends LightDaemonAnalyzerTest
CodeInsightTestFixtureImpl.ensureIndexesUpToDate(getProject());
final List<HighlightInfo> infos = new ArrayList<HighlightInfo>();
PlatformTestUtil.assertTiming("Performance failed", maxMillis, new Runnable() {
PlatformTestUtil.startPerformanceTest(maxMillis, new ThrowableRunnable() {
@Override
public void run() {
public void run() throws Exception {
infos.clear();
DaemonCodeAnalyzer.getInstance(getProject()).restart();
List<HighlightInfo> h = doHighlighting();
infos.addAll(h);
}
});
}).cpuBound().assertTiming();
return DaemonAnalyzerTestCase.filter(infos, HighlightSeverity.ERROR);
}
@@ -1,8 +1,9 @@
package com.intellij.codeInsight.daemon;
import com.intellij.testFramework.IdeaTestUtil;
import com.intellij.psi.*;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.ThrowableRunnable;
/**
* @author cdr
@@ -11,17 +12,18 @@ public class RecursiveVisitorTest extends LightDaemonAnalyzerTestCase{
public void testHugeConcatenationVisitingPerformance() throws IncorrectOperationException {
StringBuilder text = new StringBuilder("String s = \"xxx\"");
final int N = 2;
for (int i=0;i< N;i++) {
for (int i = 0; i < N; i++) {
text.append("+\"xxx\"");
}
text.append(";");
final PsiElement expression = JavaPsiFacade.getInstance(getProject()).getElementFactory().createStatementFromText(text.toString(), null);
final PsiElement expression =
JavaPsiFacade.getInstance(getProject()).getElementFactory().createStatementFromText(text.toString(), null);
final int[] n = {0};
IdeaTestUtil.assertTiming("",20,new Runnable(){
PlatformTestUtil.startPerformanceTest(20, new ThrowableRunnable() {
@Override
public void run() {
n[0]=0;
expression.accept(new JavaRecursiveElementWalkingVisitor(){
public void run() throws Exception {
n[0] = 0;
expression.accept(new JavaRecursiveElementWalkingVisitor() {
@Override
public void visitBinaryExpression(final PsiBinaryExpression expression) {
PsiExpression s = expression.getLOperand();
@@ -32,21 +34,22 @@ public class RecursiveVisitorTest extends LightDaemonAnalyzerTestCase{
});
assertEquals(N, n[0]);
}
});
}).cpuBound().assertTiming();
}
public void testHugeMethodChainingVisitingPerformance() throws IncorrectOperationException {
StringBuilder text = new StringBuilder("Object s = new StringBuilder()");
final int N = 1500;
for (int i=0;i< N;i++) {
for (int i = 0; i < N; i++) {
text.append(".append(\"xxx\")");
}
text.append(";");
final PsiElement expression = JavaPsiFacade.getInstance(getProject()).getElementFactory().createStatementFromText(text.toString(), null);
final PsiElement expression =
JavaPsiFacade.getInstance(getProject()).getElementFactory().createStatementFromText(text.toString(), null);
final int[] n = {0};
IdeaTestUtil.assertTiming("",100,new Runnable(){
PlatformTestUtil.startPerformanceTest(100, new ThrowableRunnable() {
@Override
public void run() {
n[0]=0;
public void run() throws Exception {
n[0] = 0;
expression.accept(new JavaRecursiveElementWalkingVisitor() {
@Override
public void visitMethodCallExpression(PsiMethodCallExpression expression) {
@@ -56,6 +59,6 @@ public class RecursiveVisitorTest extends LightDaemonAnalyzerTestCase{
});
assertEquals(N, n[0]);
}
});
}).cpuBound().assertTiming();
}
}
@@ -1,40 +1,41 @@
package com.intellij.openapi.vfs;
import com.intellij.testFramework.IdeaTestCase;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.testFramework.LightPlatformTestCase;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.ThrowableRunnable;
import java.io.File;
import java.io.IOException;
public class GetPathPerformanceTest extends IdeaTestCase {
public class GetPathPerformanceTest extends LightPlatformTestCase {
public void testGetPath() throws IOException, InterruptedException {
final File dir = FileUtil.createTempDirectory("GetPath","");
disposeOnTearDown(new Disposable() {
@Override
public void dispose() {
FileUtil.delete(dir);
}
});
public void testGetPath() throws IOException {
File dir = createTempDirectory();
File subdir1 = new File(dir, "1");
File subdir2 = new File(dir, "2");
subdir1.mkdir();
subdir2.mkdir();
for (int i = 0; i < 10; ++i) {
new File(subdir1, "" + i).createNewFile();
new File(subdir2, "" + i).createNewFile();
}
VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(dir.getPath().replace(File.separatorChar, '/'));
String path = dir.getPath() + StringUtil.repeat("/xxx", 50) + "/fff.txt";
File ioFile = new File(path);
boolean b = ioFile.getParentFile().mkdirs();
assertTrue(b);
boolean c = ioFile.createNewFile();
assertTrue(c);
final VirtualFile file = LocalFileSystem.getInstance().refreshAndFindFileByPath(ioFile.getPath().replace(File.separatorChar, '/'));
assertNotNull(file);
final VirtualFile[] children = file.getChildren();
for( VirtualFile child: children) {
child.getPath();
}
Runnable runnable = new Runnable() {
PlatformTestUtil.startPerformanceTest("VF.getPath() performance failed", 3000, new ThrowableRunnable() {
@Override
public void run() {
for( int i = 0; i < 1000000; ++i ) {
for( VirtualFile child: children) {
child.getPath();
}
for (int i = 0; i < 1000000; ++i) {
file.getPath();
}
}
};
PlatformTestUtil.assertTiming("Performance failed", 2000, runnable);
}).cpuBound().assertTiming();
}
}
@@ -16,6 +16,7 @@ import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.testFramework.Timings;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.Processor;
import com.intellij.util.ThrowableRunnable;
import com.intellij.util.ui.UIUtil;
import org.jetbrains.annotations.NonNls;
@@ -384,16 +385,16 @@ public class VirtualFilePointerTest extends IdeaTestCase {
}
public void testContainerDeletePerformance() throws Exception {
PlatformTestUtil.assertTiming("", 3000, new Runnable() {
PlatformTestUtil.startPerformanceTest(3000, new ThrowableRunnable() {
@Override
public void run() {
public void run() throws Exception {
Disposable parent = Disposer.newDisposable();
for (int i = 0; i < 10000; i++) {
myVirtualFilePointerManager.createContainer(parent);
}
Disposer.dispose(parent);
}
});
}).cpuBound().assertTiming();
}
private static void doVfsRefresh() {
@@ -17,8 +17,6 @@ package com.intellij.psi;
import com.intellij.lang.FileASTNode;
import com.intellij.openapi.application.ex.PathManagerEx;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.projectRoots.impl.JavaSdkImpl;
import com.intellij.openapi.roots.LanguageLevelProjectExtension;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.pom.java.LanguageLevel;
@@ -26,8 +24,9 @@ import com.intellij.psi.impl.DebugUtil;
import com.intellij.psi.impl.source.JavaFileStubBuilder;
import com.intellij.psi.impl.source.JavaLightStubBuilder;
import com.intellij.psi.stubs.StubElement;
import com.intellij.testFramework.IdeaTestUtil;
import com.intellij.testFramework.LightIdeaTestCase;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.ThrowableRunnable;
import java.io.File;
import java.security.SecureRandom;
@@ -357,12 +356,12 @@ public class JavaStubBuilderTest extends LightIdeaTestCase {
String text = FileUtil.loadFile(new File(path));
final PsiJavaFile file = (PsiJavaFile)createLightFile("test.java", text);
IdeaTestUtil.assertTiming("Source file size: " + text.length(), 2000, new Runnable() {
PlatformTestUtil.startPerformanceTest("Source file size: " + text.length(), 2000, new ThrowableRunnable() {
@Override
public void run() {
public void run() throws Exception {
NEW_BUILDER.buildStubTree(file);
}
});
}).cpuBound().assertTiming();
}
private static void doTest(final String source, final String tree) {
@@ -55,8 +55,7 @@ public class IdeaTestUtil extends PlatformTestUtil {
@SuppressWarnings({"UseOfSystemOutOrSystemErr"})
public static void printDetectedPerformanceTimings() {
System.out.println("Etalon timing: " + Timings.ETALON_TIMING);
System.out.println("This machine timing: " + Timings.MACHINE_TIMING);
System.out.println(Timings.getStatistics());
}
public static void withLevel(final Module module, final LanguageLevel level, final Runnable r) {
@@ -450,6 +450,16 @@ public abstract class LightPlatformCodeInsightTestCase extends LightPlatformTest
}
}
protected void caretUp() {
EditorActionManager actionManager = EditorActionManager.getInstance();
EditorActionHandler action = actionManager.getActionHandler(IdeActions.ACTION_EDITOR_MOVE_CARET_UP);
action.execute(getEditor(), DataManager.getInstance().getDataContext());
}
protected void deleteLine() {
EditorActionManager actionManager = EditorActionManager.getInstance();
EditorActionHandler action = actionManager.getActionHandler(IdeActions.ACTION_EDITOR_DELETE_LINE);
action.execute(getEditor(), DataManager.getInstance().getDataContext());
}
protected static void type(char c) {
EditorActionManager actionManager = EditorActionManager.getInstance();
final DataContext dataContext = DataManager.getInstance().getDataContext();
@@ -408,6 +408,9 @@ public class PlatformTestUtil {
public static TestInfo startPerformanceTest(int expected, @NotNull ThrowableRunnable test) {
return new TestInfo(test, expected);
}
public static TestInfo startPerformanceTest(@NonNls @NotNull String message, int expected, @NotNull ThrowableRunnable test) {
return startPerformanceTest(expected, test).message(message);
}
public static class TestInfo {
private final ThrowableRunnable test; // runnable to measure
private final int expected; // millis the test is expected to run
@@ -415,6 +418,8 @@ public class PlatformTestUtil {
private boolean usesAllCPUCores; // true if the test runs faster on multicore
private int attempts = 4; // number of retries if performance failed
private String message=""; // to print on fail
private boolean adjustForIO = true; // true if test uses IO, timings need to be recalibrated according to this agent disk performance
private boolean adjustForCPU = true; // true if test uses CPU, timings need to be recalibrated according to this agent CPU speed
private TestInfo(@NotNull ThrowableRunnable test, int expected) {
this.test = test;
@@ -422,34 +427,69 @@ public class PlatformTestUtil {
}
public TestInfo setup(@NotNull ThrowableRunnable setup) { assert this.setup==null; this.setup = setup; return this; }
public TestInfo usesAllCPUCores(boolean usesAllCPUCores) { this.usesAllCPUCores = usesAllCPUCores; return this; }
public TestInfo usesAllCPUCores() { assert adjustForCPU : "This test configured to be io-bound, it cannot use all cores"; usesAllCPUCores = true; return this; }
public TestInfo cpuBound() { adjustForIO = false; adjustForCPU = true; return this; }
public TestInfo ioBound() { adjustForIO = true; adjustForCPU = false; return this; }
public TestInfo attempts(int attempts) { this.attempts = attempts; return this; }
public TestInfo message(@NonNls @NotNull String message) { assert this.message==""; this.message = message; return this; }
public void assertTiming() {
assert expected != 0 : "Must call .expect() before run test";
int adjusted = usesAllCPUCores ? expected * 8 / JobSchedulerImpl.CORES_COUNT : expected;
PlatformTestUtil.assertTiming(message, adjusted, attempts, setup == null ? null : new Runnable() {
@Override
public void run() {
try {
setup.run();
}
catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
if (COVERAGE_ENABLED_BUILD) return;
while (true) {
attempts--;
long start;
try {
if (setup != null) setup.run();
start = System.currentTimeMillis();
test.run();
}
}, new Runnable() {
@Override
public void run() {
try {
test.run();
}
catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
});
long finish = System.currentTimeMillis();
long duration = finish - start;
int expectedOnMyMachine = expected;
if (adjustForCPU) {
expectedOnMyMachine = Math.max(1, (int)(1.0 * expectedOnMyMachine * Timings.CPU_TIMING / Timings.ETALON_CPU_TIMING));
expectedOnMyMachine = usesAllCPUCores ? expectedOnMyMachine * 8 / JobSchedulerImpl.CORES_COUNT : expectedOnMyMachine;
}
if (adjustForIO) {
expectedOnMyMachine = Math.max(1, (int)(1.0 * expectedOnMyMachine * Timings.IO_TIMING / Timings.ETALON_IO_TIMING));
}
final double acceptableChangeFactor = 1.1;
// Allow 10% more in case of test machine is busy.
String logMessage = message;
if (duration > expectedOnMyMachine) {
int percentage = (int)(100.0 * (duration - expectedOnMyMachine) / expectedOnMyMachine);
logMessage += ". (" + percentage + "% longer).";
}
logMessage += " Expected: " + expectedOnMyMachine + "." +
" Actual: " + duration + "." + Timings.getStatistics() ;
if (duration < expectedOnMyMachine) {
TeamCityLogger.info(logMessage);
System.out.println("SUCCESS: "+logMessage);
}
else if (duration < expectedOnMyMachine * acceptableChangeFactor) {
TeamCityLogger.warning(logMessage, null);
System.out.println("WARNING: " + logMessage);
}
else {
// try one more time
if (attempts == 0) throw new AssertionFailedError(logMessage);
System.gc();
System.gc();
System.gc();
String s = "Another epic fail (remaining attempts: " + attempts + "): " + logMessage;
TeamCityLogger.warning(s, null);
System.err.println(s);
continue;
}
break;
}
}
}
@@ -466,13 +506,8 @@ public class PlatformTestUtil {
}
public static void assertTiming(String message, long expected, int attempts, @NotNull Runnable actionToMeasure) {
assertTiming(message, expected, attempts, null, actionToMeasure);
}
public static void assertTiming(String message, long expected, int attempts, @Nullable Runnable setup, @NotNull Runnable actionToMeasure) {
while (true) {
attempts--;
if (setup != null) setup.run();
long duration = measure(actionToMeasure);
try {
assertTiming(message, expected, duration);
@@ -33,8 +33,15 @@ public class Timings {
public static final long IO_TIMING;
public static final long MACHINE_TIMING;
static {
/**
* Measured on dual core p4 3HZ 1gig ram
*/
public static final long ETALON_TIMING = 438;
public static final long ETALON_CPU_TIMING = 200;
public static final long ETALON_IO_TIMING = 100;
static {
long start = System.currentTimeMillis();
BigInteger k = new BigInteger("1");
for (int i = 0; i < CPU_PROBES; i++) {
@@ -89,18 +96,19 @@ public class Timings {
MACHINE_TIMING = CPU_TIMING + IO_TIMING;
}
/**
* Measured on dual core p4 3HZ 1gig ram
*/
public static final long ETALON_TIMING = 438;
/**
* @param value
* @return value calibrated according to this machine speed. For slower machine, lesser value will be returned
*/
public static int adjustAccordingToMySpeed(int value) {
//System.out.println("ETALON_TIMING = " + ETALON_TIMING);
//System.out.println("MACHINE_TIMING = " + MACHINE_TIMING);
return Math.max(1, (int)(1.0 * value * ETALON_TIMING / MACHINE_TIMING) / 8 * JobSchedulerImpl.CORES_COUNT);
}
public static String getStatistics() {
return
" Timings: CPU=" + CPU_TIMING + " (" + (int)(CPU_TIMING*1.0/ ETALON_CPU_TIMING*100) + "% of the etalon)" +
", I/O=" + IO_TIMING + " (" + (int)(IO_TIMING*1.0/ ETALON_IO_TIMING*100) + "% of the etalon)" +
", total=" + MACHINE_TIMING + " ("+(int)(MACHINE_TIMING*1.0/ ETALON_TIMING*100) + "% of the etalon)" +
".";
}
}
@@ -130,7 +130,7 @@ public interface CodeInsightTestFixture extends IdeaProjectTestFixture {
* @param checkWarnings
* @param checkInfos
* @param checkWeakWarnings
* @return
* @return duration
*/
long checkHighlighting(boolean checkWarnings, boolean checkInfos, boolean checkWeakWarnings);
@@ -28,8 +28,9 @@ import com.intellij.codeInspection.LocalInspectionTool;
import com.intellij.lang.ant.dom.AntResolveInspection;
import com.intellij.lang.ant.validation.AntDuplicateTargetsInspection;
import com.intellij.openapi.application.PluginPathManager;
import com.intellij.testFramework.IdeaTestUtil;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.testFramework.TestDataFile;
import com.intellij.util.ThrowableRunnable;
import org.jetbrains.annotations.NonNls;
import java.util.Collections;
@@ -110,12 +111,12 @@ public class AntHighlightingTest extends DaemonAnalyzerTestCase {
try {
myIgnoreInfos = true;
IdeaTestUtil.assertTiming("Should be quite performant !", 25000, new Runnable() {
PlatformTestUtil.startPerformanceTest("Should be quite performant !", 25000, new ThrowableRunnable() {
@Override
public void run() {
doDoTest(true, false);
}
});
}).cpuBound().assertTiming();
}
finally {
myIgnoreInfos = false;
@@ -11,6 +11,7 @@ import org.jetbrains.plugins.groovy.lang.psi.GroovyFile
import org.jetbrains.plugins.groovy.lang.psi.api.statements.typedef.members.GrMethod
import org.jetbrains.plugins.groovy.lang.psi.impl.GroovyPsiManager
import org.jetbrains.plugins.groovy.codeInspection.noReturnMethod.MissingReturnInspection
import com.intellij.util.ThrowableRunnable
/**
* @author peter
@@ -21,6 +22,13 @@ class GroovyStressTest extends LightCodeInsightFixtureTestCase {
LightGroovyTestCase.GROOVY_DESCRIPTOR
}
ThrowableRunnable highlight;
@Override
protected void setUp() {
super.setUp();
highlight = { myFixture.doHighlighting() } as ThrowableRunnable
}
public void testDontWalkLongInferenceChain() throws Exception {
Map<Integer, PsiClass> classes = [:]
myFixture.addFileToProject "Foo0.groovy", """class Foo0 {
@@ -91,7 +99,7 @@ class GroovyStressTest extends LightCodeInsightFixtureTestCase {
text += "}"
myFixture.configureByText("a.groovy", text)
IdeaTestUtil.assertTiming "slow", 5000, { myFixture.doHighlighting() }
IdeaTestUtil.startPerformanceTest("slow", 5000, highlight).cpuBound().usesAllCPUCores().assertTiming()
}
public void testDeeplyNestedClosures() {
@@ -103,7 +111,7 @@ class GroovyStressTest extends LightCodeInsightFixtureTestCase {
}
myFixture.configureByText("a.groovy", defs + text)
myFixture.enableInspections(new MissingReturnInspection())
IdeaTestUtil.assertTiming "slow", 10000, { myFixture.doHighlighting() }
IdeaTestUtil.startPerformanceTest("slow", 10000, highlight).cpuBound().usesAllCPUCores().assertTiming()
}
public void testDeeplyNestedClosuresInGenericCalls() {
@@ -113,7 +121,7 @@ class GroovyStressTest extends LightCodeInsightFixtureTestCase {
}
myFixture.configureByText("a.groovy", "def <T> foo(T t, Closure cl) {}\n" + text)
myFixture.enableInspections(new MissingReturnInspection())
IdeaTestUtil.assertTiming "slow", 10000, { myFixture.doHighlighting() }
IdeaTestUtil.startPerformanceTest("slow", 10000, highlight).cpuBound().usesAllCPUCores().assertTiming()
}
@@ -124,13 +132,12 @@ class GroovyStressTest extends LightCodeInsightFixtureTestCase {
}
myFixture.configureByText("a.groovy", "def <T> foo(T t, Closure<T> cl) {}\n" + text)
myFixture.enableInspections(new MissingReturnInspection())
IdeaTestUtil.assertTiming "slow", 10000, { myFixture.doHighlighting() }
IdeaTestUtil.startPerformanceTest("slow", 10000, highlight).cpuBound().usesAllCPUCores().assertTiming()
}
public void testManyAnnotatedScriptVariables() {
myFixture.configureByText("a.groovy", (0..100).collect { "@Anno String i$it = null" }.join("\n"))
IdeaTestUtil.assertTiming "slow", 10000, { myFixture.doHighlighting() }
IdeaTestUtil.startPerformanceTest("slow", 10000, highlight).cpuBound().usesAllCPUCores().assertTiming()
}
}
@@ -23,6 +23,7 @@ import com.intellij.spellchecker.dictionary.Loader;
import com.intellij.spellchecker.engine.Transformation;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.Consumer;
import com.intellij.util.ThrowableRunnable;
import gnu.trove.THashSet;
import junit.framework.TestCase;
import org.jetbrains.annotations.NotNull;
@@ -53,7 +54,7 @@ public class DictionaryTest extends TestCase {
}
public void testDictionary() throws IOException {
final String[] names = new String[]{JETBRAINS_DIC, ENGLISH_DIC};
final String[] names = {JETBRAINS_DIC, ENGLISH_DIC};
for (String name : names) {
loadDictionaryTest(name, sizes.get(name));
loadHalfDictionaryTest(name, 50000);
@@ -62,24 +63,23 @@ public class DictionaryTest extends TestCase {
public void loadDictionaryTest(@NotNull final String name, int wordCount) throws IOException {
final Transformation transform = new Transformation();
PlatformTestUtil.assertTiming("Dictionary load time depends on words count. Approximate word count: " + wordCount + ".", times.get(name),
new Runnable() {
@Override
public void run() {
dictionary = CompressedDictionary.create(new StreamLoader(DefaultBundledDictionariesProvider.class.getResourceAsStream(name), name), transform);
}
});
PlatformTestUtil.startPerformanceTest(times.get(name), new ThrowableRunnable() {
@Override
public void run() throws Exception {
dictionary = CompressedDictionary
.create(new StreamLoader(DefaultBundledDictionariesProvider.class.getResourceAsStream(name), name), transform);
}
}).cpuBound().assertTiming();
final Set<String> wordsToStoreAndCheck = createWordSets(name, 50000, 1).getFirst();
PlatformTestUtil.assertTiming("Invoke 'contains' " + wordsToStoreAndCheck.size() + " times", 2000, new Runnable() {
PlatformTestUtil.startPerformanceTest(2000, new ThrowableRunnable() {
@Override
public void run() {
public void run() throws Exception {
for (String s : wordsToStoreAndCheck) {
assertTrue(dictionary.contains(s));
}
}
});
}).cpuBound().assertTiming();
}
private static Loader createLoader(final Set<String> words) {