From fd2fe222be45a0a017f61846fa68228f399d3890 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 2 Dec 2021 21:35:34 +0100 Subject: [PATCH] [kotlin] performance tests migration to suite dsl Relates to #KTIJ-20250 GitOrigin-RevId: 0e9090c1b27ed961bb8959241e9ec0e2af7f03ed --- .../live/AHeavyInspectionsPerformanceTest.kt | 2 +- .../live/AbstractPerformanceProjectsTest.kt | 157 +------ .../idea/perf/live/PerformanceProjectsTest.kt | 436 ++++++++++-------- .../stats/PerformanceProjectsStatNamesTest.kt | 27 +- .../kotlin/idea/perf/stats/statsUtils.kt | 4 +- .../suite/AbstractFixtureMeasurementScope.kt | 38 ++ .../idea/perf/suite/MeasurementScope.kt | 35 ++ .../TypeAndAutoCompletionMeasurementScope.kt | 38 ++ .../suite/TypeAndHighlightMeasurementScope.kt | 34 ++ .../perf/suite/TypeAndUndoMeasurementScope.kt | 36 ++ .../kotlin/idea/perf/suite/configs.kt | 87 ++++ .../perf/{util => suite}/performanceSuite.kt | 230 ++++----- .../perf/synthetic/PerformanceStressTest.kt | 61 +-- .../HighlightWholeProjectPerformanceTest.kt | 3 + .../kotlin/idea/testFramework/Fixture.kt | 64 ++- 15 files changed, 702 insertions(+), 550 deletions(-) create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/AbstractFixtureMeasurementScope.kt create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/MeasurementScope.kt create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndAutoCompletionMeasurementScope.kt create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndHighlightMeasurementScope.kt create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndUndoMeasurementScope.kt create mode 100644 plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/configs.kt rename plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/{util => suite}/performanceSuite.kt (73%) diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AHeavyInspectionsPerformanceTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AHeavyInspectionsPerformanceTest.kt index 558e88ce8b11..b3b518bf5971 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AHeavyInspectionsPerformanceTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AHeavyInspectionsPerformanceTest.kt @@ -7,7 +7,7 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.testFramework.UsefulTestCase import org.jetbrains.kotlin.idea.perf.util.ExternalProject import org.jetbrains.kotlin.idea.perf.util.lastPathSegment -import org.jetbrains.kotlin.idea.perf.util.suite +import org.jetbrains.kotlin.idea.perf.suite.suite import org.jetbrains.kotlin.test.JUnit3RunnerWithInners import org.junit.runner.RunWith diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AbstractPerformanceProjectsTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AbstractPerformanceProjectsTest.kt index 03dc8ca400ca..f9935230617d 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AbstractPerformanceProjectsTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/AbstractPerformanceProjectsTest.kt @@ -4,7 +4,6 @@ package org.jetbrains.kotlin.idea.perf.live import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.codeInsight.daemon.impl.IdentifierHighlighterPassFactory -import com.intellij.codeInsight.lookup.LookupElement import com.intellij.codeInspection.InspectionProfileEntry import com.intellij.openapi.actionSystem.IdeActions import com.intellij.openapi.editor.EditorFactory @@ -27,9 +26,8 @@ import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.perf.ProjectBuilder import org.jetbrains.kotlin.idea.testFramework.Stats import org.jetbrains.kotlin.idea.testFramework.Stats.Companion.WARM_UP -import org.jetbrains.kotlin.idea.testFramework.Stats.Companion.runAndMeasure -import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initApp -import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initSdk +import org.jetbrains.kotlin.idea.perf.suite.PerformanceSuite.ApplicationScope.Companion.initApp +import org.jetbrains.kotlin.idea.perf.suite.PerformanceSuite.ApplicationScope.Companion.initSdk import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.search.usagesSearch.ExpressionsOfTypeProcessor @@ -38,7 +36,7 @@ import org.jetbrains.kotlin.idea.testFramework.* import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.close import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile -import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFileInEditor +import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openInEditor import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFixture import org.jetbrains.kotlin.test.KotlinRoot import java.io.File @@ -233,59 +231,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { myApplication.setDataProvider(TestDataProvider(project)) } - fun perfTypeAndAutocomplete( - stats: Stats, - fileName: String, - marker: String, - insertString: String, - surroundItems: String = "\n", - lookupElements: List, - typeAfterMarker: Boolean = true, - revertChangesAtTheEnd: Boolean = true, - note: String = "" - ) = perfTypeAndAutocomplete( - project(), stats, fileName, marker, insertString, surroundItems, - lookupElements = lookupElements, typeAfterMarker = typeAfterMarker, - revertChangesAtTheEnd = revertChangesAtTheEnd, note = note - ) - - fun perfTypeAndAutocomplete( - project: Project, - stats: Stats, - fileName: String, - marker: String, - insertString: String, - surroundItems: String = "\n", - lookupElements: List, - typeAfterMarker: Boolean = true, - revertChangesAtTheEnd: Boolean = true, - note: String = "" - ) { - assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty()) - perfTypeAndDo( - project, - fileName, - "typeAndAutocomplete", - note, - stats, - marker, - typeAfterMarker, - surroundItems, - insertString, - setupBlock = {}, - testBlock = { fixture: Fixture -> - fixture.complete() - }, - tearDownCheck = { fixture, value: Array? -> - val items = value?.map { e -> e.lookupString }?.toList() ?: emptyList() - for (lookupElement in lookupElements) { - assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement)) - } - }, - revertChangesAtTheEnd = revertChangesAtTheEnd - ) - } - fun perfTypeAndUndo( project: Project, stats: Stats, @@ -342,7 +287,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { val editor = fixture.editor val initialText = editor.document.text - updateScriptDependenciesIfNeeded(fileName, fixture) + fixture.updateScriptDependenciesIfNeeded() performanceTest { name("$typeTestPrefix ${notePrefix(note)}$fileName") @@ -427,7 +372,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { val editor = fixture.editor val initialText = editor.document.text - updateScriptDependenciesIfNeeded(fileName, fixture) + fixture.updateScriptDependenciesIfNeeded() val tasksIdx = editor.document.text.indexOf(marker) assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0) @@ -471,94 +416,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { } } - fun perfCopyAndPaste( - stats: Stats, - sourceFileName: String, - sourceInitialMarker: String? = null, - sourceFinalMarker: String? = null, - targetFileName: String, - targetInitialMarker: String? = null, - targetFinalMarker: String? = null, - note: String = "" - ) = perfCopyAndPaste( - project(), stats, - sourceFileName, sourceInitialMarker, sourceFinalMarker, - targetFileName, targetInitialMarker, targetFinalMarker, - note - ) - - fun perfCopyAndPaste( - project: Project, - stats: Stats, - sourceFileName: String, - sourceInitialMarker: String? = null, - sourceFinalMarker: String? = null, - targetFileName: String, - targetInitialMarker: String? = null, - targetFinalMarker: String? = null, - note: String = "" - ) { - performanceTest, String>, Boolean> { - name("${notePrefix(note)}$sourceFileName") - stats(stats) - warmUpIterations(8) - iterations(15) - setUp { - val fixture1 = openFixture(project, sourceFileName) - val fixture2 = openFixture(project, targetFileName) - - val initialText2 = fixture2.document.text - - updateScriptDependenciesIfNeeded(sourceFileName, fixture1) - updateScriptDependenciesIfNeeded(sourceFileName, fixture2) - - fixture1.selectMarkers(sourceInitialMarker, sourceFinalMarker) - fixture2.selectMarkers(targetInitialMarker, targetFinalMarker) - - it.setUpValue = Pair(arrayOf(fixture1, fixture2), initialText2) - } - test { - it.setUpValue?.let { setUpValue -> - val fixture1 = setUpValue.first[0] - val fixture2 = setUpValue.first[1] - it.value = fixture1.performEditorAction(IdeActions.ACTION_COPY) && - fixture2.performEditorAction(IdeActions.ACTION_PASTE) - - dispatchAllInvocationEvents() - } - } - tearDown { - try { - commitAllDocuments() - it.value?.let { performed -> - assertTrue("copy-n-paste has not performed well", performed) - // files could be different due to spaces - //assertEquals(it.setUpValue!!.first.document.text, it.setUpValue!!.second.document.text) - } - } finally { - it.setUpValue?.let { setUpValue -> - // pair.second.performEditorAction(IdeActions.ACTION_UNDO) - val fixture2 = setUpValue.first[1] - fixture2.applyText(setUpValue.second) - } - commitAllDocuments() - } - } - profilerConfig.enabled = true - } - } - - private fun updateScriptDependenciesIfNeeded( - fileName: String, - fixture: Fixture - ) { - if (isAKotlinScriptFile(fileName)) { - runAndMeasure("update script dependencies for $fileName") { - ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile) - } - } - } - protected fun perfHighlightFile( name: String, stats: Stats, @@ -596,7 +453,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { iterations(if (isWarmUp) 2 else iterations) checkStability(checkStability) setUp { - it.setUpValue = openFileInEditor(project, fileName) + it.setUpValue = openInEditor(project, fileName) } test { val file = it.setUpValue @@ -652,7 +509,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { stats(stats) warmUpIterations(20) iterations(50) - setUp { it.setUpValue = openFileInEditor(project, fileName) } + setUp { it.setUpValue = openInEditor(project, fileName) } test { ScriptConfigurationManager.updateScriptDependenciesSynchronously(it.setUpValue!!.psiFile) it.value = it.setUpValue diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/PerformanceProjectsTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/PerformanceProjectsTest.kt index 545ebda4bc2f..8eca7254eefe 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/PerformanceProjectsTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/live/PerformanceProjectsTest.kt @@ -4,15 +4,15 @@ package org.jetbrains.kotlin.idea.perf.live import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.codeInsight.daemon.impl.analysis.HighlightInfoHolder +import com.intellij.openapi.actionSystem.IdeActions import com.intellij.openapi.project.Project import com.intellij.psi.PsiFile import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.highlighter.KotlinHighlightVisitor +import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig +import org.jetbrains.kotlin.idea.perf.suite.* +import org.jetbrains.kotlin.idea.perf.util.* import org.jetbrains.kotlin.idea.testFramework.Stats.Companion.TEST_KEY -import org.jetbrains.kotlin.idea.testFramework.Stats.Companion.runAndMeasure -import org.jetbrains.kotlin.idea.perf.util.ExternalProject -import org.jetbrains.kotlin.idea.perf.util.Metric -import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite import org.jetbrains.kotlin.idea.testFramework.* import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile @@ -50,7 +50,30 @@ open class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { warmUp.warmUp(this) } - protected open fun stats(name: String) = Stats(name) + protected open fun profileConfig(): ProfilerConfig = ProfilerConfig() + + protected open fun outputConfig(): OutputConfig = OutputConfig() + + protected open fun suiteWithConfig(suiteName: String, block: PerformanceSuite.StatsScope.() -> Unit) { + suite( + suiteName, + config = StatsScopeConfig(outputConfig = outputConfig(), profilerConfig = profileConfig()), + block = block + ) + } + + private fun PerformanceSuite.StatsScope.defaultStatsConfig() { + with(config) { + warmup = 8 + iterations = 15 + } + } + + protected fun stats(name: String) = Stats(name, profilerConfig = profileConfig(), outputConfig = outputConfig()) + + protected open fun PerformanceSuite.ApplicationScope.kotlinProject(block: PerformanceSuite.ProjectScope.() -> Unit) { + project(name = "kotlin", path = ExternalProject.KOTLIN_PROJECT_PATH, openWith = GRADLE_PROJECT, block = block) + } fun testHelloWorldProject() { suite("Hello world project") { @@ -79,240 +102,267 @@ open class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProject() { - suite("Kotlin project") { - stats("kotlin project").use { - perfOpenKotlinProject(it) + suiteWithConfig("kotlin project") { + app { + warmUpProject() - val filesToHighlight = arrayOf( - "idea/idea-analysis/src/org/jetbrains/kotlin/idea/util/PsiPrecedences.kt", - "compiler/psi/src/org/jetbrains/kotlin/psi/KtElement.kt", - "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - "core/builtins/native/kotlin/Primitives.kt", + kotlinProject { + defaultStatsConfig() - "compiler/frontend/src/org/jetbrains/kotlin/cfg/ControlFlowProcessor.kt", - "compiler/frontend/src/org/jetbrains/kotlin/cfg/ControlFlowInformationProvider.kt", + val filesToHighlight = arrayOf( + //"idea/idea-analysis/src/org/jetbrains/kotlin/idea/util/PsiPrecedences.kt", + "compiler/psi/src/org/jetbrains/kotlin/psi/KtElement.kt", + "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", + "core/builtins/native/kotlin/Primitives.kt", - "compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.kt", - "compiler/backend/src/org/jetbrains/kotlin/codegen/inline/MethodInliner.kt" - ) + "compiler/frontend/cfg/src/org/jetbrains/kotlin/cfg/ControlFlowProcessor.kt", + "compiler/frontend/src/org/jetbrains/kotlin/cfg/ControlFlowInformationProvider.kt", - filesToHighlight.forEach { file -> perfHighlightFile(file, stats = it) } - filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = it) } + "compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.kt", + "compiler/backend/src/org/jetbrains/kotlin/codegen/inline/MethodInliner.kt" + ) - perfTypeAndHighlight( - it, - "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - "override fun getDeclarations(): List {", - "val q = import", - note = "in-method getDeclarations-import" - ) + profile(EmptyProfile) - perfTypeAndHighlight( - it, - "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - "override fun getDeclarations(): List {", - "val q = import", - typeAfterMarker = false, - note = "out-of-method import" - ) + filesToHighlight.forEach {fileToHighlight -> + fixture(fileToHighlight).use { fixture -> + measureHighlight(fixture, "empty profile") + } + } + + profile(DefaultProfile) + + filesToHighlight.forEach {fileToHighlight -> + fixture(fileToHighlight).use { fixture -> + measureHighlight(fixture) + } + } + + fixture("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt").use { fixture -> + with(fixture.typingConfig) { + marker = "override fun getDeclarations(): List {" + insertString = "val q = import" + typeAfterMarker = true + } + + measureTypeAndHighlight(fixture, "typeAndHighlight in-method getDeclarations-import") + + with(fixture.typingConfig) { + typeAfterMarker = false + } + + measureTypeAndHighlight(fixture, "typeAndHighlight out-of-method import") + } + } } } } fun testKotlinProjectCopyAndPaste() { - suite("Kotlin copy-and-paste") { - stats("Kotlin copy-and-paste").use { stat -> - perfOpenKotlinProjectFast(stat) + suiteWithConfig("Kotlin copy-and-paste") { + app { + warmUpProject() - perfCopyAndPaste( - stat, - sourceFileName = "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - targetFileName = "compiler/psi/src/org/jetbrains/kotlin/psi/KtImportInfo.kt" - ) + kotlinProject { + profile(DefaultProfile) + + defaultStatsConfig() + + fixture("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt").use { originalFixture -> + fixture("compiler/psi/src/org/jetbrains/kotlin/psi/KtImportInfo.kt").use { targetFixture -> + var copied = false + measure(originalFixture) { + before = { + copied = false + + targetFixture.storeText() + originalFixture.cursorConfig.select() + targetFixture.cursorConfig.select() + } + test = { + copied = originalFixture.performEditorAction(IdeActions.ACTION_COPY) && + targetFixture.performEditorAction(IdeActions.ACTION_PASTE) + + dispatchAllInvocationEvents() + } + after = { + try { + commitAllDocuments() + assertTrue("copy-n-paste has not performed well", copied) + // files could be different due to spaces + //assertEquals(it.setUpValue!!.first.document.text, it.setUpValue!!.second.document.text) + } finally { + targetFixture.restoreText() + commitAllDocuments() + } + } + } + } + } + } } } } fun testKotlinProjectCompletionKtFile() { - suite("Kotlin completion ktFile") { - stats("Kotlin completion ktFile").use { stat -> - perfOpenKotlinProjectFast(stat) + suiteWithConfig("Kotlin completion ktFile") { + app { + warmUpProject() - perfTypeAndAutocomplete( - stat, - "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - "override fun getDeclarations(): List {", - "val q = import", - lookupElements = listOf("importDirectives"), - note = "in-method getDeclarations-import" - ) + kotlinProject { + profile(DefaultProfile) - perfTypeAndAutocomplete( - stat, - "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", - "override fun getDeclarations(): List {", - "val q = import", - typeAfterMarker = false, - lookupElements = listOf("importDirectives"), - note = "out-of-method import" - ) + defaultStatsConfig() - perfTypeAndAutocomplete( - stat, - fileName = "compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.kt", - marker = "fun mapOwner(descriptor: DeclarationDescriptor): Type {", - insertString = "val b = bind", - typeAfterMarker = true, - lookupElements = listOf("bindingContext"), - note = "in-method completion for KotlinTypeMapper" - ) + fixture("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt").use { fixture -> + with(fixture.typingConfig) { + marker = "override fun getDeclarations(): List {" + insertString = "val q = import" + typeAfterMarker = true + } - perfTypeAndAutocomplete( - stat, - fileName = "compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.kt", - marker = "fun mapOwner(descriptor: DeclarationDescriptor): Type {", - insertString = "val b = bind", - typeAfterMarker = false, - lookupElements = listOf("bindingContext"), - note = "out-of-method completion for KotlinTypeMapper" - ) + measureTypeAndAutoCompletion(fixture, "in-method getDeclarations-import") { + lookupElements = listOf("importDirectives") + } - perfTypeAndAutocomplete( - stat, - fileName = "compiler/tests/org/jetbrains/kotlin/util/ArgsToParamsMatchingTest.kt", - marker = "fun testMatchNamed() {", - insertString = "testMatch", - typeAfterMarker = true, - lookupElements = listOf("testMatchNamed"), - note = "in-method completion for ArgsToParamsMatchingTest" - ) + with(fixture.typingConfig) { + typeAfterMarker = false + } - perfTypeAndAutocomplete( - stat, - fileName = "compiler/tests/org/jetbrains/kotlin/util/ArgsToParamsMatchingTest.kt", - marker = "class ArgsToParamsMatchingTest {", - insertString = "val me = ", - typeAfterMarker = true, - lookupElements = listOf("ArgsToParamsMatchingTest"), - note = "out-of-method completion for ArgsToParamsMatchingTest" - ) + measureTypeAndAutoCompletion(fixture, "out-of-method import") { + lookupElements = listOf("importDirectives") + + } + } + + fixture("compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.kt").use { fixture -> + with(fixture.typingConfig) { + marker = "fun mapOwner(descriptor: DeclarationDescriptor): Type {" + insertString = "val b = bind" + typeAfterMarker = true + } + + measureTypeAndAutoCompletion(fixture, "in-method completion for KotlinTypeMapper") { + lookupElements = listOf("bindingContext") + } + + with(fixture.typingConfig) { + typeAfterMarker = false + } + + measureTypeAndAutoCompletion(fixture, "out-of-method completion for KotlinTypeMapper") { + lookupElements = listOf("bindingContext") + } + } + + fixture("compiler/tests/org/jetbrains/kotlin/util/ArgsToParamsMatchingTest.kt").use { fixture -> + with(fixture.typingConfig) { + marker = "fun testMatchNamed() {" + insertString = "testMatch" + typeAfterMarker = true + } + + measureTypeAndAutoCompletion(fixture, "in-method completion for ArgsToParamsMatchingTest") { + lookupElements = listOf("testMatchNamed") + } + + with(fixture.typingConfig) { + typeAfterMarker = false + } + + measureTypeAndAutoCompletion(fixture, "out-of-method completion for ArgsToParamsMatchingTest") { + lookupElements = listOf("testMatchNamed") + } + } + } } } } fun testKotlinProjectCompletionBuildGradle() { - suite("Kotlin completion gradle.kts") { - stats("kotlin completion gradle.kts").use { stat -> - runAndMeasure("open kotlin project") { - perfOpenKotlinProjectFast(stat) - } + suiteWithConfig("kotlin completion gradle.kts") { + app { + warmUpProject() - runAndMeasure("type and autocomplete") { - perfTypeAndAutocomplete( - stat, - "build.gradle.kts", - "tasks {", - "reg", - lookupElements = listOf("register"), - note = "tasks-create" - ) - } + kotlinProject { + profile(DefaultProfile) - runAndMeasure("type and undo") { - perfTypeAndUndo( - project(), - stat, - "build.gradle.kts", - "tasks {", - "register", - note = "type-undo" - ) - } + defaultStatsConfig() + fixture("build.gradle.kts").use { fixture -> + with(fixture.typingConfig) { + marker = "tasks {" + insertString = "reg" + typeAfterMarker = true + } + + measureTypeAndAutoCompletion(fixture, "tasks-create") { + lookupElements = listOf("register") + } + + with(fixture.typingConfig) { + marker = "tasks {" + insertString = "register" + typeAfterMarker = true + } + + measureTypeAndUndo(fixture, "type-undo") {} + } + } } } } fun testKotlinProjectScriptDependenciesBuildGradle() { - suite("Kotlin scriptDependencies gradle.kts") { - stats("kotlin scriptDependencies gradle.kts").use { stat -> - perfOpenKotlinProjectFast(stat) + suiteWithConfig("kotlin scriptDependencies gradle.kts") { + app { + warmUpProject() - perfScriptDependenciesBuildGradleKts(stat) - perfScriptDependenciesIdeaBuildGradleKts(stat) -// perfScriptDependenciesJpsGradleKts(stat) -// perfScriptDependenciesVersionGradleKts(stat) + kotlinProject { + profile(DefaultProfile) + + defaultStatsConfig() + + fixture("build.gradle.kts", updateScriptDependenciesIfNeeded = false).use { fixture -> + measure(combineNameWithSimpleFileName("updateScriptDependencies", fixture)) { + before = { + fixture.openInEditor() + } + test = { + fixture.updateScriptDependenciesIfNeeded() + } + } + } + } } } } fun testKotlinProjectBuildGradle() { - suite("Kotlin gradle.kts") { - stats("kotlin gradle.kts").use { stat -> - perfOpenKotlinProjectFast(stat) + suiteWithConfig("kotlin gradle.kts") { + app { + warmUpProject() - perfFileAnalysisBuildGradleKts(stat) - perfFileAnalysisIdeaBuildGradleKts(stat) -// perfFileAnalysisJpsGradleKts(stat) -// perfFileAnalysisVersionGradleKts(stat) + kotlinProject { + profile(DefaultProfile) + + defaultStatsConfig() + + fixture("build.gradle.kts").use { fixture -> + measure>(combineNameWithSimpleFileName("fileAnalysis", fixture)) { + before = { + fixture.openInEditor() + } + test = { + fixture.doHighlighting() + } + } + } + } } } } - private fun perfOpenKotlinProjectFast(stats: Stats) = perfOpenKotlinProject(stats, fast = true) - - private fun perfOpenKotlinProject(stats: Stats, fast: Boolean = false) { - myProject = openProject("kotlin", stats, "", ExternalProject.KOTLIN_PROJECT_PATH, GRADLE_PROJECT, fast) - } - - protected open fun openProject( - name: String, - stats: Stats, - note: String, - path: String, - openAction: ProjectOpenAction, - fast: Boolean - ) = perfOpenProject( - name = name, - stats = stats, - note = note, - path = path, - openAction = openAction, - fast = fast - ) - - private fun perfScriptDependenciesBuildGradleKts(it: Stats) { - perfScriptDependencies("build.gradle.kts", stats = it) - } - - private fun perfScriptDependenciesIdeaBuildGradleKts(it: Stats) { - perfScriptDependencies("idea/build.gradle.kts", stats = it, note = "idea/") - } - - private fun perfScriptDependenciesJpsGradleKts(it: Stats) { - perfScriptDependencies("gradle/jps.gradle.kts", stats = it, note = "gradle/") - } - - private fun perfScriptDependenciesVersionGradleKts(it: Stats) { - perfScriptDependencies("gradle/versions.gradle.kts", stats = it, note = "gradle/") - } - - private fun perfFileAnalysisBuildGradleKts(it: Stats) { - perfKtsFileAnalysis("build.gradle.kts", stats = it) - } - - private fun perfFileAnalysisIdeaBuildGradleKts(it: Stats) { - perfKtsFileAnalysis("idea/build.gradle.kts", stats = it, note = "idea/") - } - - private fun perfFileAnalysisJpsGradleKts(it: Stats) { - perfKtsFileAnalysis("gradle/jps.gradle.kts", stats = it, note = "gradle/") - } - - private fun perfFileAnalysisVersionGradleKts(it: Stats) { - perfKtsFileAnalysis("gradle/versions.gradle.kts", stats = it, note = "gradle/") - } - private fun perfKtsFileAnalysis( fileName: String, stats: Stats, @@ -374,7 +424,7 @@ open class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } private fun replaceWithCustomHighlighter() { - org.jetbrains.kotlin.idea.testFramework.replaceWithCustomHighlighter( + replaceWithCustomHighlighter( testRootDisposable, KotlinHighlightVisitor::class.java.name, TestKotlinHighlightVisitor::class.java.name diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/PerformanceProjectsStatNamesTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/PerformanceProjectsStatNamesTest.kt index e2573a614a61..1275e027ae62 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/PerformanceProjectsStatNamesTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/PerformanceProjectsStatNamesTest.kt @@ -1,30 +1,21 @@ // Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.kotlin.idea.perf.stats -import com.intellij.openapi.project.Project -import org.jetbrains.kotlin.idea.testFramework.Stats import org.jetbrains.kotlin.idea.perf.live.PerformanceProjectsTest +import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig +import org.jetbrains.kotlin.idea.perf.util.ExternalProject +import org.jetbrains.kotlin.idea.perf.util.OutputConfig +import org.jetbrains.kotlin.idea.perf.suite.PerformanceSuite import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction class PerformanceProjectsStatNamesTest: PerformanceProjectsTest() { - override fun stats(name: String): Stats = - Stats(name, outputConfig = statsOutputConfig, profilerConfig = statsProfilerConfig) + override fun profileConfig(): ProfilerConfig = statsProfilerConfig - override fun openProject( - name: String, - stats: Stats, - note: String, - path: String, - openAction: ProjectOpenAction, - fast: Boolean - ): Project { - try { - return openProjectNormal(name, path, ProjectOpenAction.EXISTING_IDEA_PROJECT) - } catch (e: Exception) { - e.printStackTrace() - } - return openProjectNormal(name, path, openAction) + override fun outputConfig(): OutputConfig = statsOutputConfig + + override fun PerformanceSuite.ApplicationScope.kotlinProject(block: PerformanceSuite.ProjectScope.() -> Unit) { + project(name = "kotlin", path = ExternalProject.KOTLIN_PROJECT_PATH, openWith = ProjectOpenAction.EXISTING_IDEA_PROJECT, block = block) } } \ No newline at end of file diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/statsUtils.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/statsUtils.kt index d11e765d5c0b..f0e02de3dcf3 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/statsUtils.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/stats/statsUtils.kt @@ -22,9 +22,11 @@ fun compareBenchmarkWithSample(benchmark: Benchmark): Collection? { buildId = null val statsFile = statsFile() buildId = id + statsFile.name } - val sampleStatFile = File((Benchmark::class as Any).javaClass.getResource("/stats-samples/$statsFileName").path) + val resource = (Benchmark::class as Any).javaClass.getResource("/stats-samples/$statsFileName") ?: error("no sample for ${statsFileName}") + val sampleStatFile = File(resource.path) val sampleBenchmark = sampleStatFile.loadBenchmark() return compare(sampleBenchmark, benchmark).takeIf { it.isNotEmpty() } } diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/AbstractFixtureMeasurementScope.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/AbstractFixtureMeasurementScope.kt new file mode 100644 index 000000000000..0fd1a83727d8 --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/AbstractFixtureMeasurementScope.kt @@ -0,0 +1,38 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import org.jetbrains.kotlin.idea.testFramework.Fixture +import org.jetbrains.kotlin.idea.testFramework.Stats +import org.jetbrains.kotlin.idea.testFramework.commitAllDocuments + +abstract class AbstractFixtureMeasurementScope( + protected val fixture: Fixture, + typeTestPrefix: String = "", + name: String, + stats: Stats, + config: StatsScopeConfig, + after: (() -> Unit)? = null, + var revertChangesAtTheEnd: Boolean = true, +) : MeasurementScope(listOf(typeTestPrefix, name).filter { it.isNotEmpty() }.joinToString(" "), stats, config, after = after) { + protected fun doFixturePerformanceTest(setUp: () -> Unit = {}, test:() -> V?, tearDown: () -> Unit = {}) { + doPerformanceTest( + setUp = { + fixture.storeText() + setUp() + before.invoke() + }, + test = test, + tearDown = { + try { + tearDown() + } finally { + if (revertChangesAtTheEnd) { + fixture.restoreText() + commitAllDocuments() + } + after?.invoke() + } + } + ) + } +} \ No newline at end of file diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/MeasurementScope.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/MeasurementScope.kt new file mode 100644 index 000000000000..a3060f33697b --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/MeasurementScope.kt @@ -0,0 +1,35 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import org.jetbrains.kotlin.idea.testFramework.Stats +import org.jetbrains.kotlin.idea.testFramework.performanceTest + +open class MeasurementScope( + val name: String, + val stats: Stats, + val config: StatsScopeConfig, + var before: () -> Unit = {}, + var test: (() -> T?)? = null, + var after: (() -> Unit)? = null +) { + open fun run(): List { + val t = test ?: error("test procedure isn't set") + val value = mutableListOf() + doPerformanceTest(before, test = { value.add(t.invoke()) }, tearDown = after?: {}) + return value + } + + protected fun doPerformanceTest(setUp: () -> Unit = {}, test:() -> V?, tearDown: () -> Unit = {}) { + performanceTest { + name(name) + stats(stats) + warmUpIterations(config.warmup) + iterations(config.iterations) + fastIterations(config.fastIterations) + setUp { setUp() } + test { test() } + tearDown { tearDown() } + profilerConfig(config.profilerConfig) + } + } +} diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndAutoCompletionMeasurementScope.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndAutoCompletionMeasurementScope.kt new file mode 100644 index 000000000000..871e22092fdd --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndAutoCompletionMeasurementScope.kt @@ -0,0 +1,38 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import com.intellij.codeInsight.lookup.LookupElement +import junit.framework.TestCase.assertTrue +import org.jetbrains.kotlin.idea.testFramework.Fixture +import org.jetbrains.kotlin.idea.testFramework.Stats + +class TypeAndAutoCompletionMeasurementScope( + fixture: Fixture, + typeTestPrefix: String = "", + name: String, + stats: Stats, + config: StatsScopeConfig, + after: (() -> Unit)? = null, + var lookupElements: List = listOf(), + revertChangesAtTheEnd: Boolean = true, +): AbstractFixtureMeasurementScope(fixture, typeTestPrefix, name, stats, config, after, revertChangesAtTheEnd) { + override fun run(): List { + assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty()) + var value: Array? = null + doFixturePerformanceTest( + setUp = { + fixture.typingConfig.moveCursor() + fixture.type() + }, + test = { value = fixture.complete() }, + tearDown = { + val items = value?.map { e -> e.lookupString }?.toList() ?: emptyList() + for (lookupElement in lookupElements) { + assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement)) + } + } + ) + return value?.map { e -> e.lookupString }?.toList() ?: emptyList() + } + +} diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndHighlightMeasurementScope.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndHighlightMeasurementScope.kt new file mode 100644 index 000000000000..d44780d67898 --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndHighlightMeasurementScope.kt @@ -0,0 +1,34 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import com.intellij.codeInsight.daemon.impl.HighlightInfo +import com.intellij.testFramework.UsefulTestCase.assertNotEmpty +import org.jetbrains.kotlin.idea.testFramework.Fixture +import org.jetbrains.kotlin.idea.testFramework.Stats + +class TypeAndHighlightMeasurementScope( + fixture: Fixture, + typeTestPrefix: String = "", + name: String, + stats: Stats, + config: StatsScopeConfig, + after: (() -> Unit)? = null, + revertChangesAtTheEnd: Boolean = true, +) : AbstractFixtureMeasurementScope(fixture, typeTestPrefix, name, stats, config, after, revertChangesAtTheEnd) { + override fun run(): List { + var highlightInfos: List? = null + doFixturePerformanceTest( + setUp = { + fixture.typingConfig.moveCursor() + }, + test = { + highlightInfos = fixture.typeAndHighlight() + highlightInfos + }, + tearDown = { + highlightInfos?.let { assertNotEmpty(it) } + } + ) + return highlightInfos ?: emptyList() + } +} \ No newline at end of file diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndUndoMeasurementScope.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndUndoMeasurementScope.kt new file mode 100644 index 000000000000..4884df49d9a6 --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/TypeAndUndoMeasurementScope.kt @@ -0,0 +1,36 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import com.intellij.openapi.actionSystem.IdeActions +import com.intellij.util.ui.UIUtil +import org.jetbrains.kotlin.idea.testFramework.Fixture +import org.jetbrains.kotlin.idea.testFramework.Stats + +class TypeAndUndoMeasurementScope( + fixture: Fixture, + typeTestPrefix: String = "", + name: String, + stats: Stats, + config: StatsScopeConfig, + after: (() -> Unit)? = null, + revertChangesAtTheEnd: Boolean = true, +) : AbstractFixtureMeasurementScope(fixture, typeTestPrefix, name, stats, config, after, revertChangesAtTheEnd) { + override fun run(): List { + doFixturePerformanceTest( + setUp = { + fixture.typingConfig.moveCursor() + fixture.type() + }, + test = { + fixture.performEditorAction(IdeActions.ACTION_UNDO) + UIUtil.dispatchAllInvocationEvents() + }, + tearDown = { + val text = fixture.document.text + val savedText = fixture.savedText + assert(savedText != text) { "undo has to change document text\nbefore undo:\n$savedText\n\nafter undo:\n$text" } + } + ) + return listOf(fixture.document.text) + } +} \ No newline at end of file diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/configs.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/configs.kt new file mode 100644 index 000000000000..52081de61120 --- /dev/null +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/configs.kt @@ -0,0 +1,87 @@ +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.kotlin.idea.perf.suite + +import com.intellij.testFramework.EditorTestUtil +import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig +import org.jetbrains.kotlin.idea.perf.util.ExternalProject +import org.jetbrains.kotlin.idea.perf.util.OutputConfig +import org.jetbrains.kotlin.idea.perf.util.lastPathSegment +import org.jetbrains.kotlin.idea.testFramework.Fixture +import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction + +data class CursorConfig( + val fixture: Fixture, + var marker: String? = null, + var finalMarker: String? = null +) : AutoCloseable { + + fun select() { + fixture.selectMarkers(marker, finalMarker) + } + + override fun close() { + fixture.close() + } +} + +data class TypingConfig( + val fixture: Fixture, + /** + * Find the place in a file and place cursor at [marker], + * If [typeAfterMarker] is true cursor is placed after the [marker], + * otherwise at the beggining of the [marker]. + */ + var marker: String? = null, + var typeAfterMarker: Boolean = true, + var insertString: String? = null, + var surroundItems: String = "\n", + var note: String = "", + var delayMs: Long? = null +) : AutoCloseable { + fun moveCursor() { + val editor = fixture.editor + + val tasksIdx = marker?.let { marker -> + fixture.text.indexOf(marker).also { + check(it > 0) { "marker '$marker' not found in ${fixture.fileName}" } + } + } ?: 0 + if (typeAfterMarker || marker == null) { + editor.caretModel.moveToOffset(tasksIdx + (marker?.let { it.length + 1 } ?: 0)) + } else { + editor.caretModel.moveToOffset(tasksIdx - 1) + } + + for (surroundItem in surroundItems) { + EditorTestUtil.performTypingAction(editor, surroundItem) + } + + editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2) + + if (!typeAfterMarker) { + for (surroundItem in surroundItems) { + EditorTestUtil.performTypingAction(editor, surroundItem) + } + editor.caretModel.moveToOffset(editor.caretModel.offset - 2) + } + } + + override fun close() { + fixture.close() + } +} + +class StatsScopeConfig( + var name: String? = null, + var warmup: Int = 2, + var iterations: Int = 5, + var fastIterations: Boolean = false, + var outputConfig: OutputConfig = OutputConfig(), + var profilerConfig: ProfilerConfig = ProfilerConfig() +) + +class ProjectScopeConfig(val path: String, val openWith: ProjectOpenAction, val refresh: Boolean = false, private val name: String? = null) { + val projectName: String = name ?: path.lastPathSegment() + + constructor(externalProject: ExternalProject, refresh: Boolean) : this(externalProject.path, externalProject.openWith, refresh) +} \ No newline at end of file diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/performanceSuite.kt similarity index 73% rename from plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt rename to plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/performanceSuite.kt index 645c4383b38d..3953351543ff 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/suite/performanceSuite.kt @@ -1,6 +1,6 @@ // Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package org.jetbrains.kotlin.idea.perf.util +package org.jetbrains.kotlin.idea.perf.suite import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.lang.annotation.HighlightSeverity @@ -19,7 +19,6 @@ import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess import com.intellij.psi.PsiDocumentManager import com.intellij.psi.PsiFile -import com.intellij.psi.PsiManager import com.intellij.psi.impl.PsiDocumentManagerBase import com.intellij.psi.util.PsiTreeUtil import com.intellij.testFramework.* @@ -34,7 +33,7 @@ import org.jetbrains.kotlin.idea.framework.KotlinSdkType import org.jetbrains.kotlin.idea.perf.ProjectBuilder import org.jetbrains.kotlin.idea.testFramework.Stats import org.jetbrains.kotlin.idea.testFramework.Stats.Companion.runAndMeasure -import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig +import org.jetbrains.kotlin.idea.perf.util.* import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.disableAllInspections import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableAllInspections import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableInspections @@ -43,6 +42,7 @@ import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultPro import org.jetbrains.kotlin.idea.test.GradleProcessOutputInterceptor import org.jetbrains.kotlin.idea.test.invalidateLibraryCache import org.jetbrains.kotlin.idea.testFramework.* +import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.close import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe import org.jetbrains.kotlin.psi.KtDeclaration @@ -51,7 +51,6 @@ import java.io.File class PerformanceSuite { companion object { - fun suite( name: String, stats: StatsScope, @@ -97,67 +96,21 @@ class PerformanceSuite { fun measure(name: String, f: MeasurementScope.() -> Unit, after: (() -> Unit)?): List = MeasurementScope(name, stats, config, after = after).apply(f).run() + fun measureTypeAndAutoCompletion(name: String, fixture: Fixture, f: TypeAndAutoCompletionMeasurementScope.() -> Unit, after: (() -> Unit)?): List = + TypeAndAutoCompletionMeasurementScope(fixture, typeTestPrefix = "typeAndAutocomplete", name = name, stats = stats, config = config, after = after).apply(f).run() + + fun measureTypeAndUndo(name: String, fixture: Fixture, f: TypeAndUndoMeasurementScope.() -> Unit, after: (() -> Unit)?): List = + TypeAndUndoMeasurementScope(fixture, typeTestPrefix = "typeAndUndo", name = name, stats = stats, config = config, after = after).apply(f).run() + + fun measureTypeAndHighlight(name: String, fixture: Fixture, f: TypeAndHighlightMeasurementScope.() -> Unit, after: (() -> Unit)?): List = + TypeAndHighlightMeasurementScope(fixture, typeTestPrefix = "", name = name, stats = stats, config = config, after = after).apply(f).run() + fun logStatValue(name: String, value: Any) { logMessage { "buildStatisticValue key='${stats.name}: $name' value='$value'" } TeamCity.statValue("${stats.name}: $name", value) } } - data class CursorConfig( - val fixture: Fixture, - val marker: String - ) : AutoCloseable { - override fun close() { - fixture.close() - } - } - - data class TypingConfig( - val fixture: Fixture, - val marker: String, - val insertString: String, - val surroundItems: String = "\n", - val typeAfterMarker: Boolean = true, - val note: String = "", - val delayMs: Long? = null - ) : AutoCloseable { - override fun close() { - fixture.close() - } - } - - class MeasurementScope( - val name: String, - val stats: Stats, - val config: StatsScopeConfig, - var before: () -> Unit = {}, - var test: (() -> T?)? = null, - var after: (() -> Unit)? = null - ) { - fun run(): List { - val t = test ?: error("test procedure isn't set") - val value = mutableListOf() - performanceTest { - name(name) - stats(stats) - warmUpIterations(config.warmup) - iterations(config.iterations) - fastIterations(config.fastIterations) - setUp { - before() - } - test { - value.add(t.invoke()) - } - tearDown { - after?.invoke() - } - profilerConfig(config.profilerConfig) - } - return value - } - } - class ApplicationScope(val rootDisposable: Disposable, val stats: StatsScope) : AutoCloseable { val application = initApp(rootDisposable) val jdk: Sdk = initSdk(rootDisposable) @@ -168,11 +121,11 @@ class PerformanceSuite { fun project(block: ProjectWithDescriptorScope.() -> Unit) = ProjectWithDescriptorScope(this).use(block) - fun project(path: String, openWith: ProjectOpenAction = ProjectOpenAction.EXISTING_IDEA_PROJECT, block: ProjectScope.() -> Unit) = - ProjectScope(ProjectScopeConfig(path, openWith), this).use(block) + fun project(name: String? = null, path: String, openWith: ProjectOpenAction = ProjectOpenAction.EXISTING_IDEA_PROJECT, block: ProjectScope.() -> Unit) = + ProjectScope(ProjectScopeConfig(path, openWith, name = name), this).use(block) - fun gradleProject(path: String, refresh: Boolean = false, block: ProjectScope.() -> Unit) = - ProjectScope(ProjectScopeConfig(path, ProjectOpenAction.GRADLE_PROJECT, refresh), this).use(block) + fun gradleProject(name: String? = null, path: String, refresh: Boolean = false, block: ProjectScope.() -> Unit) = + ProjectScope(ProjectScopeConfig(path, ProjectOpenAction.GRADLE_PROJECT, refresh, name = name), this).use(block) fun warmUpProject() = project { descriptor { @@ -235,22 +188,6 @@ class PerformanceSuite { } } - - class StatsScopeConfig( - var name: String? = null, - var warmup: Int = 2, - var iterations: Int = 5, - var fastIterations: Boolean = false, - var outputConfig: OutputConfig = OutputConfig(), - var profilerConfig: ProfilerConfig = ProfilerConfig() - ) - - class ProjectScopeConfig(val path: String, val openWith: ProjectOpenAction, val refresh: Boolean = false) { - val name: String = path.lastPathSegment() - - constructor(externalProject: ExternalProject, refresh: Boolean) : this(externalProject.path, externalProject.openWith, refresh) - } - abstract class AbstractProjectScope(val app: ApplicationScope) : AutoCloseable { abstract val project: Project val openFiles = mutableListOf() @@ -276,58 +213,6 @@ class PerformanceSuite { fun highlight(editorFile: PsiFile?, toIgnore: IntArray = ArrayUtilRt.EMPTY_INT_ARRAY) = editorFile?.highlightFile(toIgnore) ?: error("editor isn't ready for highlight") - fun moveCursor(config: CursorConfig) { - val fixture = config.fixture - val editor = fixture.editor - updateScriptDependenciesIfNeeded(fixture) - - val marker = config.marker - val tasksIdx = fixture.text.indexOf(marker) - check(tasksIdx > 0) { - "marker '$marker' not found in ${fixture.fileName}" - } - editor.caretModel.moveToOffset(tasksIdx) - } - - fun moveCursor(config: TypingConfig) { - val fixture = config.fixture - val editor = fixture.editor - updateScriptDependenciesIfNeeded(fixture) - - val marker = config.marker - val tasksIdx = fixture.text.indexOf(marker) - check(tasksIdx > 0) { - "marker '$marker' not found in ${fixture.fileName}" - } - if (config.typeAfterMarker) { - editor.caretModel.moveToOffset(tasksIdx + marker.length + 1) - } else { - editor.caretModel.moveToOffset(tasksIdx - 1) - } - - for (surroundItem in config.surroundItems) { - EditorTestUtil.performTypingAction(editor, surroundItem) - } - - editor.caretModel.moveToOffset(editor.caretModel.offset - if (config.typeAfterMarker) 1 else 2) - - if (!config.typeAfterMarker) { - for (surroundItem in config.surroundItems) { - EditorTestUtil.performTypingAction(editor, surroundItem) - } - editor.caretModel.moveToOffset(editor.caretModel.offset - 2) - } - } - - fun typeAndHighlight(config: TypingConfig): List { - val string = config.insertString - for (i in string.indices) { - config.fixture.type(string[i]) - config.delayMs?.let { d -> Thread.sleep(d) } - } - return config.fixture.doHighlighting() - } - fun findUsages(config: CursorConfig): Set { val offset = config.fixture.editor.caretModel.offset val psiFile = config.fixture.psiFile @@ -353,24 +238,24 @@ class PerformanceSuite { this.project.enableAllInspections() fun editor(path: String) = - Fixture.openFileInEditor(project, path).psiFile.also { openFiles.add(it.virtualFile) } + Fixture.openInEditor(project, path).psiFile.also { openFiles.add(it.virtualFile) } - fun fixture(path: String): Fixture { - return fixture(Fixture.projectFileByName(project, path).virtualFile) + fun fixture(path: String, updateScriptDependenciesIfNeeded: Boolean = true): Fixture { + return fixture(Fixture.projectFileByName(project, path).virtualFile, updateScriptDependenciesIfNeeded) } - fun fixture(file: VirtualFile): Fixture { + fun fixture(file: VirtualFile, updateScriptDependenciesIfNeeded: Boolean = true): Fixture { val fixture = Fixture.openFixture(project, file) openFiles.add(fixture.vFile) val fileName = file.name - if (Fixture.isAKotlinScriptFile(fileName)) { - ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile) - } if (fileName.endsWith(KotlinFileType.EXTENSION)) { assert(fixture.psiFile is KtFile) { "$file expected to be a Kotlin file" } } + if (updateScriptDependenciesIfNeeded) { + fixture.updateScriptDependenciesIfNeeded() + } return fixture } @@ -385,12 +270,71 @@ class PerformanceSuite { } fun measure(vararg name: String, clearCaches: Boolean = true, f: MeasurementScope.() -> Unit): List { - val after = { if (clearCaches) PsiManager.getInstance(project).dropPsiCaches() } + val after = wrapAfter(clearCaches) return app.stats.measure(name.joinToString("-"), f, after) } - fun measure(vararg name: String, fixture: Fixture, f: MeasurementScope.() -> Unit): List = - measure("${name.joinToString("-")} ${fixture.fileName}", f = f) + private fun wrapAfter(clearCaches: Boolean): () -> Unit { + val after = if (clearCaches) { + fun() { project.cleanupCaches() } + } else { + fun() {} + } + return after + } + + fun measure(fixture: Fixture, f: MeasurementScope.() -> Unit): List = + measure(fixture.fileName, f = f) + + fun measure(fixture: Fixture, vararg name: String, f: MeasurementScope.() -> Unit): List = + measure(combineName(fixture, *name), f = f) + + fun measureTypeAndHighlight( + fixture: Fixture, + vararg name: String, + f: TypeAndHighlightMeasurementScope.() -> Unit = {} + ): List { + val after = wrapAfter(true) + return app.stats.measureTypeAndHighlight(combineName(fixture, *name), fixture, f, after) + } + + fun measureHighlight(fixture: Fixture, vararg name: String): List?> { + return measure(combineNameWithSimpleFileName("highlighting", fixture, *name)) { + before = { + fixture.openInEditor() + } + test = { + val document = FileDocumentManager.getInstance().getDocument(fixture.vFile)!! + val editor = EditorFactory.getInstance().getEditors(document).first() + commitAllDocuments() + CodeInsightTestFixtureImpl.instantiateAndRun(fixture.psiFile, editor, ArrayUtilRt.EMPTY_INT_ARRAY, true) + } + after = { + fixture.close() + project.cleanupCaches() + } + } + } + + fun measureTypeAndAutoCompletion(fixture: Fixture, vararg name: String, clearCaches: Boolean = true, f: TypeAndAutoCompletionMeasurementScope.() -> Unit): List { + val after = wrapAfter(clearCaches) + return app.stats.measureTypeAndAutoCompletion(combineName(fixture, *name), fixture, f, after) + } + + fun measureTypeAndUndo(fixture: Fixture, vararg name: String, clearCaches: Boolean = true, f: TypeAndUndoMeasurementScope.() -> Unit): List { + val after = wrapAfter(clearCaches) + return app.stats.measureTypeAndUndo(combineName(fixture, *name), fixture, f, after) + } + + fun combineName(fixture: Fixture, vararg name: String) = + listOf(name.joinToString("-"), fixture.simpleFilename()) + .filter { it.isNotEmpty() } + .joinToString(" ") + + fun combineNameWithSimpleFileName(type: String, fixture: Fixture, vararg name: String): String = + listOf(type, name.joinToString("-"), fixture.simpleFilename()) + .filter { it.isNotEmpty() } + .joinToString(" ") override fun close(): Unit = RunAll( { compilerTester?.tearDown() }, @@ -425,7 +369,7 @@ class PerformanceSuite { val openProject = OpenProject( projectPath = projectPath, - projectName = config.name, + projectName = config.projectName, jdk = app.jdk, projectOpenAction = config.openWith ) @@ -464,7 +408,7 @@ data class CustomProfile(val inspectionNames: List) : ProjectProfile() fun UsefulTestCase.suite( suiteName: String? = null, - config: PerformanceSuite.StatsScopeConfig = PerformanceSuite.StatsScopeConfig(), + config: StatsScopeConfig = StatsScopeConfig(), block: PerformanceSuite.StatsScope.() -> Unit ) { val stats = Stats(config.name ?: suiteName ?: name, outputConfig = config.outputConfig, profilerConfig = config.profilerConfig) diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/synthetic/PerformanceStressTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/synthetic/PerformanceStressTest.kt index 0a9b5e026995..8ecaad344349 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/synthetic/PerformanceStressTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/synthetic/PerformanceStressTest.kt @@ -2,17 +2,17 @@ package org.jetbrains.kotlin.idea.perf.synthetic -import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.testFramework.UsefulTestCase import com.intellij.usages.Usage -import junit.framework.TestCase import org.jetbrains.kotlin.idea.perf.Parameter import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig +import org.jetbrains.kotlin.idea.perf.suite.DefaultProfile +import org.jetbrains.kotlin.idea.perf.suite.PerformanceSuite +import org.jetbrains.kotlin.idea.perf.suite.StatsScopeConfig +import org.jetbrains.kotlin.idea.perf.suite.suite import org.jetbrains.kotlin.idea.perf.util.* -import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.TypingConfig import org.jetbrains.kotlin.idea.perf.util.registerLoadingErrorsHeadlessNotifier import org.jetbrains.kotlin.idea.test.IDEA_TEST_DATA_DIR -import org.jetbrains.kotlin.idea.testFramework.commitAllDocuments import org.jetbrains.kotlin.test.JUnit3RunnerWithInners import org.junit.runner.RunWith @@ -23,6 +23,14 @@ open class PerformanceStressTest : UsefulTestCase() { protected open fun outputConfig(): OutputConfig = OutputConfig() + protected open fun suiteWithConfig(suiteName: String, name: String? = null, block: PerformanceSuite.StatsScope.() -> Unit) { + suite( + suiteName, + config = StatsScopeConfig(name = name, outputConfig = outputConfig(), profilerConfig = profileConfig()), + block = block + ) + } + override fun setUp() { super.setUp() @@ -44,10 +52,7 @@ open class PerformanceStressTest : UsefulTestCase() { val numberOfPackagesWithCandidates = 50 val name = "findUsages${numberOfFuns}_$numberOfPackagesWithCandidates" + if (withCompilerIndex) "_with_cri" else "" - suite( - suiteName = name, - config = PerformanceSuite.StatsScopeConfig(name = name, outputConfig = outputConfig(), profilerConfig = profileConfig()), - ) { + suiteWithConfig(name) { app { warmUpProject() @@ -96,30 +101,27 @@ open class PerformanceStressTest : UsefulTestCase() { } fixture("src/pkg1/DataClass.kt").use { fixture -> - val typingConfig = PerformanceSuite.CursorConfig(fixture, marker = "DataClass") + with(fixture.cursorConfig) { marker = "DataClass" } with(config) { warmup = 8 iterations = 15 } - measure>("findUsages", fixture = fixture) { + measure>(fixture, "findUsages") { before = { - moveCursor(typingConfig) + fixture.moveCursor() } test = { - val findUsages = findUsages(typingConfig) + val findUsages = findUsages(fixture.cursorConfig) // 1 from import // + numberOfUsages as function argument // + numberOfUsages as return type functions // + numberOfUsages as new instance in a body of function // in a SomeService - TestCase.assertEquals(1 + 3 * numberOfFuns, findUsages.size) + assertEquals(1 + 3 * numberOfFuns, findUsages.size) findUsages } - after = { - - } } } } @@ -132,10 +134,7 @@ open class PerformanceStressTest : UsefulTestCase() { val generatedTypes = mutableListOf(listOf()) generateTypes(arrayOf("Int", "String", "Long", "List", "Array"), generatedTypes) - suite( - suiteName = "Lots of overloaded method project", - config = PerformanceSuite.StatsScopeConfig(name = "kt-35135 project", outputConfig = outputConfig(), profilerConfig = profileConfig()), - ) { + suiteWithConfig("Lots of overloaded method project", "kt-35135 project") { app { warmUpProject() @@ -177,30 +176,18 @@ open class PerformanceStressTest : UsefulTestCase() { profile(DefaultProfile) fixture("src/main/java/pkg/SomeClass.kt").use { fixture -> - val typingConfig = TypingConfig( - fixture, - marker = "ov", - insertString = "override fun foo(): String = TODO()", + with(fixture.typingConfig) { + marker = "ov" + insertString = "override fun foo(): String = TODO()" delayMs = 50 - ) + } with(config) { warmup = 8 iterations = 15 } - measure>("type override fun foo()", fixture = fixture) { - before = { - moveCursor(typingConfig) - } - test = { - typeAndHighlight(typingConfig) - } - after = { - fixture.restoreText() - commitAllDocuments() - } - } + measureTypeAndHighlight(fixture, "type override fun foo()") } } } diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/whole/HighlightWholeProjectPerformanceTest.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/whole/HighlightWholeProjectPerformanceTest.kt index 4205460f0a8e..6b2bbc42a0a0 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/whole/HighlightWholeProjectPerformanceTest.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/perf/whole/HighlightWholeProjectPerformanceTest.kt @@ -11,6 +11,9 @@ import com.intellij.psi.search.FileTypeIndex import com.intellij.psi.search.GlobalSearchScope import com.intellij.testFramework.UsefulTestCase import org.jetbrains.kotlin.idea.KotlinFileType +import org.jetbrains.kotlin.idea.perf.suite.DefaultProfile +import org.jetbrains.kotlin.idea.perf.suite.EmptyProfile +import org.jetbrains.kotlin.idea.perf.suite.suite import org.jetbrains.kotlin.idea.perf.util.* import org.jetbrains.kotlin.idea.search.usagesSearch.ExpressionsOfTypeProcessor import org.jetbrains.kotlin.idea.testFramework.relativePath diff --git a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt index fe40cbe934c0..51f91bcba5be 100644 --- a/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt +++ b/plugins/kotlin/performance-tests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt @@ -47,6 +47,8 @@ import org.jetbrains.kotlin.idea.KotlinLanguage import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager import org.jetbrains.kotlin.idea.core.util.toPsiFile +import org.jetbrains.kotlin.idea.perf.suite.CursorConfig +import org.jetbrains.kotlin.idea.perf.suite.TypingConfig import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase import org.jetbrains.kotlin.parsing.KotlinParserDefinition @@ -59,7 +61,11 @@ class Fixture( ) : AutoCloseable { private var delegate = EditorTestFixture(project, editor, vFile) - private var savedText: String? = null + var savedText: String? = null + get() = field + private set(value) { + field = value + } val document: Document get() = editor.document @@ -71,6 +77,11 @@ class Fixture( storeText() } + fun simpleFilename(): String { + val lastIndexOf = fileName.lastIndexOf('/') + return if (lastIndexOf >= 0) fileName.substring(lastIndexOf + 1) else fileName + } + fun doHighlighting(): List = delegate.doHighlighting() fun findUsages(psiElement: PsiElement): Set { @@ -79,6 +90,14 @@ class Fixture( return findUsagesManager.doFindUsages(arrayOf(psiElement), emptyArray(), handler, handler.findUsagesOptions, false).usages } + fun type() { + val string = typingConfig.insertString ?: error("insertString has to be specified") + for (i in string.indices) { + type(string[i]) + typingConfig.delayMs?.let(Thread::sleep) + } + } + fun type(s: String) { delegate.type(s) } @@ -87,6 +106,21 @@ class Fixture( delegate.type(c) } + fun typeAndHighlight(): List { + type() + return doHighlighting() + } + + fun moveCursor() { + val tasksIdx = cursorConfig.marker?.let { marker -> + text.indexOf(marker).also { + check(it > 0) { "marker '$marker' not found in ${fileName}" } + } + } ?: 0 + + editor.caretModel.moveToOffset(tasksIdx) + } + fun performEditorAction(actionId: String): Boolean { selectEditor() return delegate.performEditorAction(actionId) @@ -144,11 +178,27 @@ class Fixture( dispatchAllInvocationEvents() } + fun openInEditor() { + openInEditor(project, vFile) + } + + fun updateScriptDependenciesIfNeeded() { + if (isAKotlinScriptFile(fileName)) { + Stats.runAndMeasure("update script dependencies for $fileName") { + ScriptConfigurationManager.updateScriptDependenciesSynchronously(psiFile) + } + } + } + override fun close() { savedText = null project.close(vFile) } + val cursorConfig = CursorConfig(this) + + val typingConfig = TypingConfig(this) + companion object { // quite simple impl - good so far fun isAKotlinScriptFile(fileName: String) = fileName.endsWith(KotlinParserDefinition.STD_SCRIPT_EXT) @@ -195,7 +245,7 @@ class Fixture( fun openFixture(project: Project, fileName: String): Fixture { - val fileInEditor = openFileInEditor(project, fileName) + val fileInEditor = openInEditor(project, fileName) val file = fileInEditor.psiFile val editorFactory = EditorFactory.getInstance() val editor = editorFactory.getEditors(fileInEditor.document, project)[0] @@ -204,7 +254,7 @@ class Fixture( } fun openFixture(project: Project, file: VirtualFile): Fixture { - val fileInEditor = openFileInEditor(project, file) + val fileInEditor = openInEditor(project, file) val psiFile = fileInEditor.psiFile val editorFactory = EditorFactory.getInstance() val editor = editorFactory.getEditors(fileInEditor.document, project)[0] @@ -242,12 +292,12 @@ class Fixture( return virtualFiles.iterator().next().toPsiFile(project)!! } - fun openFileInEditor(project: Project, name: String): EditorFile { + fun openInEditor(project: Project, name: String): EditorFile { val psiFile = projectFileByName(project, name) - return openFileInEditor(project, psiFile.virtualFile) + return openInEditor(project, psiFile.virtualFile) } - fun openFileInEditor(project: Project, vFile: VirtualFile): EditorFile { + fun openInEditor(project: Project, vFile: VirtualFile): EditorFile { val fileDocumentManager = FileDocumentManager.getInstance() val fileEditorManager = FileEditorManager.getInstance(project) @@ -281,7 +331,7 @@ class Fixture( lookupElements: List, revertChangesAtTheEnd: Boolean = true ) { - val fileInEditor = openFileInEditor(project, fileName) + val fileInEditor = openInEditor(project, fileName) val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0] val fixture = Fixture(fileName, project, editor, fileInEditor.psiFile)