From e16f6da212c194a3fcdc5ce5420bfa628a57f4a0 Mon Sep 17 00:00:00 2001 From: Roman Vasiliev Date: Thu, 10 Oct 2024 14:44:41 +0200 Subject: [PATCH] [evaluation-plugin] LME-66 Eliminate separate EvaluationDataset GitOrigin-RevId: f80ddd7c222b76864cf0414427573160fa20ca79 --- .../ContextCollectionEvaluationCommand.kt | 18 +++---- .../{CsvDataset.kt => CsvEnvironment.kt} | 27 +++++----- .../intellij/cce/actions/EvaluationDataset.kt | 39 --------------- ...ataset.kt => ProjectActionsEnvironment.kt} | 47 +++++++++++++++--- .../cce/evaluable/EvaluableFeatureBase.kt | 27 +++++----- .../cce/evaluable/StandaloneFeature.kt | 12 ----- ...flictDataset.kt => ConflictEnvironment.kt} | 20 +++----- .../ConflictResolutionFeature.kt | 4 +- .../StandaloneExampleFeature.kt | 8 +-- .../ActionsInterpretationHandler.kt | 8 ++- .../cce/evaluation/BackgroundStepFactory.kt | 10 ++-- .../cce/evaluation/EvaluationChunk.kt | 22 +++++++++ .../cce/evaluation/EvaluationEnvironment.kt | 43 ++++++++++++---- .../cce/evaluation/ProjectEnvironment.kt | 49 ------------------- .../step/ActionsInterpretationStep.kt | 7 ++- .../evaluation/step/DatasetPreparationStep.kt | 8 +-- 16 files changed, 151 insertions(+), 198 deletions(-) rename plugins/evaluation-plugin/src/com/intellij/cce/actions/{CsvDataset.kt => CsvEnvironment.kt} (83%) delete mode 100644 plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt rename plugins/evaluation-plugin/src/com/intellij/cce/actions/{ProjectActionsDataset.kt => ProjectActionsEnvironment.kt} (87%) rename plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/{ConflictDataset.kt => ConflictEnvironment.kt} (89%) create mode 100644 plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationChunk.kt delete mode 100644 plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt index 0f7e956f2aa4..2e382cc1c3a0 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt @@ -58,21 +58,19 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter. val workspace = EvaluationWorkspace.create(config, SetupStatsCollectorStep.statsCollectorLogsDirectory) val evaluationRootInfo = EvaluationRootInfo(true) feature.prepareEnvironment(config).use { environment -> - val dataset = environment.dataset + check(environment is ProjectActionsEnvironment) - check(dataset is ProjectActionsDataset) - - val actions = dataset.config - val newDataset = object : ProjectActionsDataset( + val actions = environment.config + val newEnvironment = object : ProjectActionsEnvironment( config.strategy, actions, config.interpret.filesLimit, config.interpret.sessionsLimit, evaluationRootInfo, - dataset.project, - dataset.processor, + environment.project, + environment.processor, feature.name, - dataset.featureInvoker + environment.featureInvoker ) { override fun prepare(datasetContext: DatasetContext, progress: Progress) { val files = runReadAction { @@ -84,10 +82,6 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter. } } - val newEnvironment = object : EvaluationEnvironment by environment { - override val dataset: EvaluationDataset = newDataset - } - val datasetContext = DatasetContext(workspace, workspace, null) val stepFactory = BackgroundStepFactory( diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvEnvironment.kt similarity index 83% rename from plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt rename to plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvEnvironment.kt index 283729386ea1..41beae1d4cdb 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvEnvironment.kt @@ -4,33 +4,28 @@ import com.intellij.cce.core.Session import com.intellij.cce.core.SimpleTokenProperties import com.intellij.cce.core.SymbolLocation import com.intellij.cce.core.TypeProperty -import com.intellij.cce.evaluation.EvaluationStep +import com.intellij.cce.evaluation.EvaluationChunk +import com.intellij.cce.evaluation.SimpleFileEnvironment import com.intellij.cce.interpreter.* -import com.intellij.cce.util.Progress +import java.nio.file.Path import kotlin.io.path.extension import kotlin.io.path.isRegularFile import kotlin.io.path.readLines -class CsvDataset( - private val datasetRef: DatasetRef, +class CsvEnvironment( + override val datasetRef: DatasetRef, private val chunkSize: Int, private val targetField: String, private val featureInvoker: FeatureInvoker, -) : EvaluationDataset { - override val setupSdk: EvaluationStep? = null - override val checkSdk: EvaluationStep? = null +) : SimpleFileEnvironment() { override val preparationDescription: String = "Checking that CSV file exists" - override fun prepare(datasetContext: DatasetContext, progress: Progress) { - val datasetPath = datasetContext.path(datasetRef) - + override fun checkFile(datasetPath: Path) { require(datasetPath.extension == "csv") { "Csv dataset should have the appropriate extension: $datasetRef" } - datasetRef.prepare(datasetContext) - check(datasetPath.isRegularFile()) { "$datasetRef didn't create a file: $datasetPath" } @@ -38,13 +33,13 @@ class CsvDataset( override fun sessionCount(datasetContext: DatasetContext): Int = datasetContext.path(datasetRef).readLines().size - 1 - override fun chunks(datasetContext: DatasetContext): Iterator { + override fun chunks(datasetContext: DatasetContext): Iterator { val lines = datasetContext.path(datasetRef).readLines() val dataLines = lines.subList(1, lines.size) val names = lines.first().split(',').map { it.trim() } var offset = 0 - val result = mutableListOf() + val result = mutableListOf() for (rows in dataLines.chunked(chunkSize)) { val presentationText = StringBuilder() val calls = mutableListOf() @@ -64,8 +59,8 @@ class CsvDataset( offset += 1 } - result += object : EvaluationDatasetChunk { - override val datasetName: String = this@CsvDataset.datasetRef.name + result += object : EvaluationChunk { + override val datasetName: String = this@CsvEnvironment.datasetRef.name override val name: String = "$datasetName:${offset - rows.size + 1}-${offset}" override val presentationText: String = presentationText.toString() diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt deleted file mode 100644 index 5c0dd126b745..000000000000 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt +++ /dev/null @@ -1,39 +0,0 @@ -package com.intellij.cce.actions - -import com.intellij.cce.core.Session -import com.intellij.cce.evaluation.EvaluationStep -import com.intellij.cce.interpreter.FeatureInvoker -import com.intellij.cce.interpreter.InterpretFilter -import com.intellij.cce.interpreter.InterpretationHandler -import com.intellij.cce.interpreter.InterpretationOrder -import com.intellij.cce.util.Progress - -/** - * Represents data which will be used for evaluation. - */ -interface EvaluationDataset { // FIXME with implicit invoker inside it may make sense to merge this interface into EvaluationEnvironment - val setupSdk: EvaluationStep? - val checkSdk: EvaluationStep? - - val preparationDescription: String - - fun prepare(datasetContext: DatasetContext, progress: Progress) - - fun sessionCount(datasetContext: DatasetContext): Int - - // TODO should return something closeable for large files - fun chunks(datasetContext: DatasetContext): Iterator -} - -interface EvaluationDatasetChunk { - val datasetName: String - val name: String - val presentationText: String - - fun evaluate( - handler: InterpretationHandler, - filter: InterpretFilter, - order: InterpretationOrder, - sessionHandler: (Session) -> Unit - ): List -} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsEnvironment.kt similarity index 87% rename from plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt rename to plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsEnvironment.kt index edcc47d383dd..1dd5a2ecb13a 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsEnvironment.kt @@ -4,10 +4,9 @@ package com.intellij.cce.actions import com.intellij.cce.core.* import com.intellij.cce.evaluable.EvaluationStrategy import com.intellij.cce.evaluable.common.CommonActionsInvoker -import com.intellij.cce.evaluation.EvaluationRootInfo -import com.intellij.cce.evaluation.EvaluationStep -import com.intellij.cce.evaluation.SetupSdkStep +import com.intellij.cce.evaluation.* import com.intellij.cce.evaluation.step.CheckProjectSdkStep +import com.intellij.cce.evaluation.step.runInIntellij import com.intellij.cce.interpreter.* import com.intellij.cce.processor.DefaultEvaluationRootProcessor import com.intellij.cce.processor.EvaluationRootByRangeProcessor @@ -19,15 +18,19 @@ import com.intellij.cce.util.Summary import com.intellij.cce.util.text import com.intellij.cce.visitor.CodeFragmentBuilder import com.intellij.cce.workspace.Config +import com.intellij.cce.workspace.EvaluationWorkspace import com.intellij.cce.workspace.info.FileErrorInfo import com.intellij.cce.workspace.storages.storage.ActionsSingleFileStorage +import com.intellij.ide.impl.runUnderModalProgressIfIsEdt import com.intellij.openapi.application.ReadAction import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.VirtualFile +import com.intellij.warmup.util.importOrOpenProjectAsync +import java.nio.file.FileSystems import kotlin.random.Random -open class ProjectActionsDataset( +open class ProjectActionsEnvironment( private val strategy: EvaluationStrategy, val config: Config.ActionsGeneration, private val filesLimit: Int?, // TODO dataset generation could be lazy @@ -37,7 +40,7 @@ open class ProjectActionsDataset( val processor: GenerateActionsProcessor, private val featureName: String, val featureInvoker: FeatureInvoker, -) : EvaluationDataset { +) : EvaluationEnvironment { private val datasetRef = config.sourceFile.run { val sf = this ?: "" if (sf.isNotBlank()) { @@ -89,7 +92,7 @@ open class ProjectActionsDataset( return datasetContext.actionsStorage.computeSessionsCount() } - override fun chunks(datasetContext: DatasetContext): Iterator { + override fun chunks(datasetContext: DatasetContext): Iterator { ensureDataRefIsHandled(datasetContext) val files = datasetContext.actionsStorage.getActionFiles() return files.shuffled(FILES_RANDOM).asSequence().map { file -> @@ -170,6 +173,13 @@ open class ProjectActionsDataset( actionsSummarizer.save(datasetContext) } + override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? = + step.runInIntellij(project, workspace) + + override fun close() { + ProjectOpeningUtils.closeProject(project) + } + private class ActionsSummarizer { private val rootSummary: Summary = Summary.create() fun update(fileActions: FileActions) { @@ -232,7 +242,7 @@ open class ProjectActionsDataset( private inner class FileActionsChunk( private val fileActions: FileActions, override val presentationText: String, - ) : EvaluationDatasetChunk { + ) : EvaluationChunk { override val datasetName: String = config.projectName override val name: String = fileActions.path @@ -250,7 +260,28 @@ open class ProjectActionsDataset( return actionInterpreter.interpret(fileActions, sessionHandler) } } + + companion object { + fun open(projectPath: String, init: (Project) -> ProjectActionsEnvironment): ProjectActionsEnvironment { + println("Open and load project $projectPath. Operation may take a few minutes.") + @Suppress("DEPRECATION") + val project = runUnderModalProgressIfIsEdt { + importOrOpenProjectAsync(OpenProjectArgsData(FileSystems.getDefault().getPath(projectPath))) + } + println("Project loaded!") + + val environment = try { + init(project) + } + catch (exception: Exception) { + ProjectOpeningUtils.closeProject(project) + throw RuntimeException("Failed to initialize project environment: $exception", exception) + } + + return environment + } + } } -private val LOG = Logger.getInstance(ProjectActionsDataset::class.java) +private val LOG = Logger.getInstance(ProjectActionsEnvironment::class.java) private val FILES_RANDOM = Random(42) diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt index 38786415f269..cef5b8ebf784 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt @@ -1,8 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.cce.evaluable -import com.intellij.cce.actions.DatasetRef -import com.intellij.cce.actions.ProjectActionsDataset +import com.intellij.cce.actions.ProjectActionsEnvironment import com.intellij.cce.core.Language import com.intellij.cce.evaluation.* import com.intellij.cce.interpreter.FeatureInvoker @@ -44,19 +43,17 @@ abstract class EvaluableFeatureBase(override val name: S override fun prepareEnvironment(config: Config): EvaluationEnvironment { val actions = actions(config) val strategy = config.strategy() - return ProjectEnvironment.open(actions.projectPath) { project -> - StandaloneEnvironment( - dataset = ProjectActionsDataset( - strategy, - actions, - config.interpret.filesLimit, - config.interpret.sessionsLimit, - EvaluationRootInfo(true), - project, - getGenerateActionsProcessor(strategy, project), - name, - featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy) - ) + return ProjectActionsEnvironment.open(actions.projectPath) { project -> + ProjectActionsEnvironment( + strategy, + actions, + config.interpret.filesLimit, + config.interpret.sessionsLimit, + EvaluationRootInfo(true), + project, + getGenerateActionsProcessor(strategy, project), + name, + featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy) ) } } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt index 68d3b3d4647d..60cab067387b 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt @@ -1,10 +1,6 @@ package com.intellij.cce.evaluable -import com.intellij.cce.actions.EvaluationDataset -import com.intellij.cce.evaluation.EvaluationEnvironment import com.intellij.cce.evaluation.EvaluationStep -import com.intellij.cce.evaluation.StandaloneEnvironment -import com.intellij.cce.interpreter.FeatureInvoker import com.intellij.cce.report.BasicFileReportGenerator import com.intellij.cce.report.FileReportGenerator import com.intellij.cce.report.GeneratorDirectories @@ -16,8 +12,6 @@ abstract class StandaloneFeature( override val name: String ) : EvaluableFeature { - abstract fun getDataset(config: Config): EvaluationDataset - override fun getPreliminaryEvaluationSteps(): List = emptyList() override fun getEvaluationSteps(config: Config): List = emptyList() @@ -29,10 +23,4 @@ abstract class StandaloneFeature( fullLineStorages: List, dirs: GeneratorDirectories ): FileReportGenerator = BasicFileReportGenerator(filterName, comparisonFilterName, featuresStorages, dirs) - - override fun prepareEnvironment(config: Config): EvaluationEnvironment { - return StandaloneEnvironment( - getDataset(config) - ) - } } \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictEnvironment.kt similarity index 89% rename from plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt rename to plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictEnvironment.kt index 618586618855..84bcf8ad9844 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictEnvironment.kt @@ -3,32 +3,26 @@ package com.intellij.cce.evaluable.conflictResolution import com.intellij.cce.actions.* import com.intellij.cce.core.* import com.intellij.cce.evaluable.AIA_CONTEXT -import com.intellij.cce.evaluation.EvaluationStep +import com.intellij.cce.evaluation.EvaluationChunk +import com.intellij.cce.evaluation.SimpleFileEnvironment import com.intellij.cce.interpreter.InterpretFilter import com.intellij.cce.interpreter.InterpretationHandler import com.intellij.cce.interpreter.InterpretationOrder -import com.intellij.cce.util.Progress import java.nio.file.Path import kotlin.io.path.listDirectoryEntries import kotlin.io.path.readText -class ConflictDataset( - private val datasetRef: DatasetRef, +class ConflictEnvironment( + override val datasetRef: DatasetRef, private val conflictResolver: ConflictResolver<*> -) : EvaluationDataset { - override val setupSdk: EvaluationStep? = null - override val checkSdk: EvaluationStep? = null +) : SimpleFileEnvironment() { override val preparationDescription: String = "Checking that conflict dataset exists" - override fun prepare(datasetContext: DatasetContext, progress: Progress) { - datasetRef.prepare(datasetContext) - } - override fun sessionCount(datasetContext: DatasetContext): Int = datasetContext.path(datasetRef).listDirectoryEntries().size // FIXME should be pre-calculated - override fun chunks(datasetContext: DatasetContext): Iterator { + override fun chunks(datasetContext: DatasetContext): Iterator { return datasetContext.path(datasetRef).listDirectoryEntries().map { conflictPath -> ConflictChunk(conflictPath) }.iterator() @@ -36,7 +30,7 @@ class ConflictDataset( private inner class ConflictChunk( private val conflictPath: Path, - ) : EvaluationDatasetChunk { + ) : EvaluationChunk { override val datasetName: String = datasetRef.name override val name: String = conflictPath.fileName.toString() override val presentationText: String = readText("result") diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt index e13ad2353af5..be1e0987ac83 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt @@ -1,9 +1,9 @@ package com.intellij.cce.evaluable.conflictResolution import com.intellij.cce.actions.DatasetRef -import com.intellij.cce.actions.EvaluationDataset import com.intellij.cce.evaluable.StandaloneFeature import com.intellij.cce.evaluable.StrategySerializer +import com.intellij.cce.evaluation.SimpleFileEnvironment import com.intellij.cce.metric.Metric import com.intellij.cce.metric.PrecisionMetric import com.intellij.cce.metric.SessionsCountMetric @@ -12,7 +12,7 @@ import com.intellij.cce.workspace.Config class ConflictResolutionFeature : StandaloneFeature("conflict-resolution") { override fun getStrategySerializer(): StrategySerializer = ConflictResolutionStrategy.Serializer() - override fun getDataset(config: Config): EvaluationDataset = ConflictDataset( + override fun prepareEnvironment(config: Config): SimpleFileEnvironment = ConflictEnvironment( DatasetRef.parse(config.conflictDataset!!.url), TheirConflictResolver() ) diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt index 6118c79682a3..3c5b991baa25 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt @@ -2,12 +2,12 @@ package com.intellij.cce.evaluable.standaloneExample import com.google.gson.JsonObject import com.google.gson.JsonSerializationContext -import com.intellij.cce.actions.CsvDataset +import com.intellij.cce.actions.CsvEnvironment import com.intellij.cce.actions.DatasetRef -import com.intellij.cce.actions.EvaluationDataset import com.intellij.cce.evaluable.EvaluationStrategy import com.intellij.cce.evaluable.StandaloneFeature import com.intellij.cce.evaluable.StrategySerializer +import com.intellij.cce.evaluation.EvaluationEnvironment import com.intellij.cce.filter.EvaluationFilter import com.intellij.cce.metric.Metric import com.intellij.cce.metric.PrecisionMetric @@ -22,9 +22,9 @@ class StandaloneExampleFeature : StandaloneFeature("standalone- override fun deserialize(map: Map, language: String): DatasetStrategy = DatasetStrategy() } - override fun getDataset(config: Config): EvaluationDataset { + override fun prepareEnvironment(config: Config): EvaluationEnvironment { val fileDataset = config.csvDataset ?: throw IllegalStateException("Required dataset config") - return CsvDataset( + return CsvEnvironment( datasetRef = DatasetRef.parse(fileDataset.url), chunkSize = fileDataset.chunkSize, targetField = "Type", diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt index 7f2c410339cb..4bbbccaa0225 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt @@ -2,9 +2,7 @@ package com.intellij.cce.evaluation import com.intellij.cce.actions.DatasetContext -import com.intellij.cce.actions.EvaluationDataset import com.intellij.cce.evaluation.step.SetupStatsCollectorStep -import com.intellij.cce.interpreter.FeatureInvoker import com.intellij.cce.interpreter.InterpretFilter import com.intellij.cce.interpreter.InterpretationHandlerImpl import com.intellij.cce.util.ExceptionsUtil @@ -35,10 +33,10 @@ class ActionsInterpretationHandler( logsSaverIf(config.interpret.saveFusLogs) { workspace.fusLogsSaver } ).asCompositeLogsSaver() - fun invoke(dataset: EvaluationDataset, workspace: EvaluationWorkspace, indicator: Progress) { + fun invoke(environment: EvaluationEnvironment, workspace: EvaluationWorkspace, indicator: Progress) { var sessionsCount: Int val computingTime = measureTimeMillis { - sessionsCount = dataset.sessionCount(datasetContext) + sessionsCount = environment.sessionCount(datasetContext) } LOG.info("Computing of sessions count took $computingTime ms") val interpretationConfig = config.interpret @@ -55,7 +53,7 @@ class ActionsInterpretationHandler( println("During actions interpretation will be skipped about $skippedSessions sessions") } var fileCount = 0 - for (chunk in dataset.chunks(datasetContext)) { + for (chunk in environment.chunks(datasetContext)) { if (config.interpret.filesLimit?.let { it <= fileCount } == true) { break } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt index b43c4b451a88..c5e745d66cce 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt @@ -17,17 +17,17 @@ class BackgroundStepFactory( private val datasetContext: DatasetContext ) : StepFactory { - override fun generateActionsStep(): EvaluationStep = DatasetPreparationStep(environment.dataset, datasetContext) + override fun generateActionsStep(): EvaluationStep = DatasetPreparationStep(environment, datasetContext) override fun interpretActionsStep(): EvaluationStep = - ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = false) + ActionsInterpretationStep(config, environment, datasetContext, newWorkspace = false) override fun generateReportStep(): EvaluationStep = ReportGenerationStep(inputWorkspacePaths?.map { EvaluationWorkspace.open(it, SetupStatsCollectorStep.statsCollectorLogsDirectory) }, config.reports.sessionsFilters, config.reports.comparisonFilters, feature) override fun interpretActionsOnNewWorkspaceStep(): EvaluationStep = - ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = true) + ActionsInterpretationStep(config, environment, datasetContext, newWorkspace = true) override fun reorderElements(): EvaluationStep = ReorderElementsStep(config) @@ -39,9 +39,9 @@ class BackgroundStepFactory( SetupStatsCollectorStep(config.interpret.experimentGroup, config.interpret.logLocationAndItemText) else null - override fun setupSdkStep(): EvaluationStep? = environment.dataset.setupSdk + override fun setupSdkStep(): EvaluationStep? = environment.setupSdk - override fun checkSdkConfiguredStep(): EvaluationStep? = environment.dataset.checkSdk + override fun checkSdkConfiguredStep(): EvaluationStep? = environment.checkSdk override fun finishEvaluationStep(): FinishEvaluationStep = HeadlessFinishEvaluationStep() diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationChunk.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationChunk.kt new file mode 100644 index 000000000000..e7eeb6d1cc51 --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationChunk.kt @@ -0,0 +1,22 @@ +package com.intellij.cce.evaluation + +import com.intellij.cce.core.Session +import com.intellij.cce.interpreter.InterpretFilter +import com.intellij.cce.interpreter.InterpretationHandler +import com.intellij.cce.interpreter.InterpretationOrder + +/** + * Represents a part of an evaluation which can be handled and represented separately. + */ +interface EvaluationChunk { + val datasetName: String + val name: String + val presentationText: String + + fun evaluate( + handler: InterpretationHandler, + filter: InterpretFilter, + order: InterpretationOrder, + sessionHandler: (Session) -> Unit + ): List +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt index 3cc045f6a582..eb07b5fbf9fd 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt @@ -1,27 +1,50 @@ package com.intellij.cce.evaluation -import com.intellij.cce.actions.EvaluationDataset +import com.intellij.cce.actions.DatasetContext +import com.intellij.cce.actions.DatasetRef import com.intellij.cce.evaluation.step.runInIntellij -import com.intellij.cce.interpreter.FeatureInvoker +import com.intellij.cce.util.Progress import com.intellij.cce.workspace.EvaluationWorkspace +import java.nio.file.Path /** * Environment represents resources needed for an evaluation. - * For example, it can be IntelliJ project like in [ProjectEnvironment] if an evaluation needs an opened project. + * For example, it can be IntelliJ project like in ProjectActionsEnvironment if an evaluation needs an opened project. * It should be initialized before an evaluation process and closed right after finish. */ interface EvaluationEnvironment : AutoCloseable { - val dataset: EvaluationDataset + + val setupSdk: EvaluationStep? + val checkSdk: EvaluationStep? + + val preparationDescription: String + + fun prepare(datasetContext: DatasetContext, progress: Progress) + + fun sessionCount(datasetContext: DatasetContext): Int + + // TODO should return something closeable for large files + fun chunks(datasetContext: DatasetContext): Iterator fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? } -/** - * A special type of environment which doesn't imply any associated resources and can be treated like a simple data class. - */ -class StandaloneEnvironment( - override val dataset: EvaluationDataset, -) : EvaluationEnvironment { +abstract class SimpleFileEnvironment : EvaluationEnvironment { + + protected abstract val datasetRef: DatasetRef + + override val setupSdk: EvaluationStep? = null + override val checkSdk: EvaluationStep? = null + + protected open fun checkFile(datasetPath: Path) { + } + + override fun prepare(datasetContext: DatasetContext, progress: Progress) { + datasetRef.prepare(datasetContext) + val datasetPath = datasetContext.path(datasetRef) + checkFile(datasetPath) + } + override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? = step.runInIntellij(null, workspace) diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt deleted file mode 100644 index 3c75662ab10f..000000000000 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt +++ /dev/null @@ -1,49 +0,0 @@ -package com.intellij.cce.evaluation - -import com.intellij.cce.actions.EvaluationDataset -import com.intellij.cce.actions.OpenProjectArgsData -import com.intellij.cce.actions.ProjectOpeningUtils -import com.intellij.cce.evaluation.step.runInIntellij -import com.intellij.cce.interpreter.FeatureInvoker -import com.intellij.cce.workspace.EvaluationWorkspace -import com.intellij.ide.impl.runUnderModalProgressIfIsEdt -import com.intellij.openapi.project.Project -import com.intellij.warmup.util.importOrOpenProjectAsync -import java.nio.file.FileSystems - -class ProjectEnvironment( - val project: Project, - override val dataset: EvaluationDataset, -) : EvaluationEnvironment { - - override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? = - step.runInIntellij(project, workspace) - - override fun close() { - ProjectOpeningUtils.closeProject(project) - } - - companion object { - fun open(projectPath: String, init: (Project) -> StandaloneEnvironment): ProjectEnvironment { - println("Open and load project $projectPath. Operation may take a few minutes.") - @Suppress("DEPRECATION") - val project = runUnderModalProgressIfIsEdt { - importOrOpenProjectAsync(OpenProjectArgsData(FileSystems.getDefault().getPath(projectPath))) - } - println("Project loaded!") - - val environment = try { - init(project) - } - catch (exception: Exception) { - ProjectOpeningUtils.closeProject(project) - throw RuntimeException("Failed to initialize project environment: $exception", exception) - } - - return ProjectEnvironment( - project, - environment.dataset - ) - } - } -} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt index cf793b88f3c6..940b89eadb8f 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt @@ -2,16 +2,15 @@ package com.intellij.cce.evaluation.step import com.intellij.cce.actions.DatasetContext -import com.intellij.cce.actions.EvaluationDataset import com.intellij.cce.evaluation.ActionsInterpretationHandler -import com.intellij.cce.interpreter.FeatureInvoker +import com.intellij.cce.evaluation.EvaluationEnvironment import com.intellij.cce.util.Progress import com.intellij.cce.workspace.Config import com.intellij.cce.workspace.EvaluationWorkspace class ActionsInterpretationStep( private val config: Config, - private val dataset: EvaluationDataset, + private val environment: EvaluationEnvironment, private val datasetContext: DatasetContext, private val newWorkspace: Boolean ) : BackgroundEvaluationStep { @@ -23,7 +22,7 @@ class ActionsInterpretationStep( val resultWorkspace = if (newWorkspace) EvaluationWorkspace.create(config, SetupStatsCollectorStep.statsCollectorLogsDirectory) else workspace - ActionsInterpretationHandler(config, datasetContext).invoke(dataset, resultWorkspace, progress) + ActionsInterpretationHandler(config, datasetContext).invoke(environment, resultWorkspace, progress) return resultWorkspace } } \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/DatasetPreparationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/DatasetPreparationStep.kt index c31db8715922..b5c754145dc5 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/DatasetPreparationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/DatasetPreparationStep.kt @@ -1,20 +1,20 @@ package com.intellij.cce.evaluation.step import com.intellij.cce.actions.DatasetContext -import com.intellij.cce.actions.EvaluationDataset +import com.intellij.cce.evaluation.EvaluationEnvironment import com.intellij.cce.util.Progress import com.intellij.cce.workspace.EvaluationWorkspace class DatasetPreparationStep( - private val dataset: EvaluationDataset, + private val environment: EvaluationEnvironment, private val datasetContext: DatasetContext, ) : BackgroundEvaluationStep { override val name: String = "Preparing dataset" - override val description: String = dataset.preparationDescription + override val description: String = environment.preparationDescription override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace { - dataset.prepare(datasetContext, progress) + environment.prepare(datasetContext, progress) return workspace } } \ No newline at end of file