[evaluation-plugin] LME-66 Eliminate separate EvaluationDataset

GitOrigin-RevId: f80ddd7c222b76864cf0414427573160fa20ca79

plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt

@@ -58,21 +58,19 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter.
     val workspace = EvaluationWorkspace.create(config, SetupStatsCollectorStep.statsCollectorLogsDirectory)
     val evaluationRootInfo = EvaluationRootInfo(true)
     feature.prepareEnvironment(config).use { environment ->
-      val dataset = environment.dataset
+      check(environment is ProjectActionsEnvironment)
 
-      check(dataset is ProjectActionsDataset)
-
-      val actions = dataset.config
-      val newDataset = object : ProjectActionsDataset(
+      val actions = environment.config
+      val newEnvironment = object : ProjectActionsEnvironment(
         config.strategy,
         actions,
         config.interpret.filesLimit,
         config.interpret.sessionsLimit,
         evaluationRootInfo,
-        dataset.project,
-        dataset.processor,
+        environment.project,
+        environment.processor,
         feature.name,
-        dataset.featureInvoker
+        environment.featureInvoker
       ) {
         override fun prepare(datasetContext: DatasetContext, progress: Progress) {
           val files = runReadAction {
@@ -84,10 +82,6 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter.
         }
       }
 
-      val newEnvironment = object : EvaluationEnvironment by environment {
-        override val dataset: EvaluationDataset = newDataset
-      }
-
       val datasetContext = DatasetContext(workspace, workspace, null)
 
       val stepFactory = BackgroundStepFactory(

plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvEnvironment.kt

@@ -4,33 +4,28 @@ import com.intellij.cce.core.Session
 import com.intellij.cce.core.SimpleTokenProperties
 import com.intellij.cce.core.SymbolLocation
 import com.intellij.cce.core.TypeProperty
-import com.intellij.cce.evaluation.EvaluationStep
+import com.intellij.cce.evaluation.EvaluationChunk
+import com.intellij.cce.evaluation.SimpleFileEnvironment
 import com.intellij.cce.interpreter.*
-import com.intellij.cce.util.Progress
+import java.nio.file.Path
 import kotlin.io.path.extension
 import kotlin.io.path.isRegularFile
 import kotlin.io.path.readLines
 
-class CsvDataset(
-  private val datasetRef: DatasetRef,
+class CsvEnvironment(
+  override val datasetRef: DatasetRef,
   private val chunkSize: Int,
   private val targetField: String,
   private val featureInvoker: FeatureInvoker,
-) : EvaluationDataset {
-  override val setupSdk: EvaluationStep? = null
-  override val checkSdk: EvaluationStep? = null
+) : SimpleFileEnvironment() {
 
   override val preparationDescription: String = "Checking that CSV file exists"
 
-  override fun prepare(datasetContext: DatasetContext, progress: Progress) {
-    val datasetPath = datasetContext.path(datasetRef)
-
+  override fun checkFile(datasetPath: Path) {
     require(datasetPath.extension == "csv") {
       "Csv dataset should have the appropriate extension: $datasetRef"
     }
 
-    datasetRef.prepare(datasetContext)
-
     check(datasetPath.isRegularFile()) {
       "$datasetRef didn't create a file: $datasetPath"
     }
@@ -38,13 +33,13 @@ class CsvDataset(
 
   override fun sessionCount(datasetContext: DatasetContext): Int = datasetContext.path(datasetRef).readLines().size - 1
 
-  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationDatasetChunk> {
+  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationChunk> {
     val lines = datasetContext.path(datasetRef).readLines()
     val dataLines = lines.subList(1, lines.size)
     val names = lines.first().split(',').map { it.trim() }
 
     var offset = 0
-    val result = mutableListOf<EvaluationDatasetChunk>()
+    val result = mutableListOf<EvaluationChunk>()
     for (rows in dataLines.chunked(chunkSize)) {
       val presentationText = StringBuilder()
       val calls = mutableListOf<CallFeature>()
@@ -64,8 +59,8 @@ class CsvDataset(
         offset += 1
       }
 
-      result += object : EvaluationDatasetChunk {
-        override val datasetName: String = this@CsvDataset.datasetRef.name
+      result += object : EvaluationChunk {
+        override val datasetName: String = this@CsvEnvironment.datasetRef.name
         override val name: String = "$datasetName:${offset - rows.size + 1}-${offset}"
         override val presentationText: String = presentationText.toString()
 

plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt

@@ -1,39 +0,0 @@
-package com.intellij.cce.actions
-
-import com.intellij.cce.core.Session
-import com.intellij.cce.evaluation.EvaluationStep
-import com.intellij.cce.interpreter.FeatureInvoker
-import com.intellij.cce.interpreter.InterpretFilter
-import com.intellij.cce.interpreter.InterpretationHandler
-import com.intellij.cce.interpreter.InterpretationOrder
-import com.intellij.cce.util.Progress
-
-/**
- * Represents data which will be used for evaluation.
- */
-interface EvaluationDataset { // FIXME with implicit invoker inside it may make sense to merge this interface into EvaluationEnvironment
-  val setupSdk: EvaluationStep?
-  val checkSdk: EvaluationStep?
-
-  val preparationDescription: String
-
-  fun prepare(datasetContext: DatasetContext, progress: Progress)
-
-  fun sessionCount(datasetContext: DatasetContext): Int
-
-  // TODO should return something closeable for large files
-  fun chunks(datasetContext: DatasetContext): Iterator<EvaluationDatasetChunk>
-}
-
-interface EvaluationDatasetChunk {
-  val datasetName: String
-  val name: String
-  val presentationText: String
-
-  fun evaluate(
-    handler: InterpretationHandler,
-    filter: InterpretFilter,
-    order: InterpretationOrder,
-    sessionHandler: (Session) -> Unit
-  ): List<Session>
-}

plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsEnvironment.kt

@@ -4,10 +4,9 @@ package com.intellij.cce.actions
 import com.intellij.cce.core.*
 import com.intellij.cce.evaluable.EvaluationStrategy
 import com.intellij.cce.evaluable.common.CommonActionsInvoker
-import com.intellij.cce.evaluation.EvaluationRootInfo
-import com.intellij.cce.evaluation.EvaluationStep
-import com.intellij.cce.evaluation.SetupSdkStep
+import com.intellij.cce.evaluation.*
 import com.intellij.cce.evaluation.step.CheckProjectSdkStep
+import com.intellij.cce.evaluation.step.runInIntellij
 import com.intellij.cce.interpreter.*
 import com.intellij.cce.processor.DefaultEvaluationRootProcessor
 import com.intellij.cce.processor.EvaluationRootByRangeProcessor
@@ -19,15 +18,19 @@ import com.intellij.cce.util.Summary
 import com.intellij.cce.util.text
 import com.intellij.cce.visitor.CodeFragmentBuilder
 import com.intellij.cce.workspace.Config
+import com.intellij.cce.workspace.EvaluationWorkspace
 import com.intellij.cce.workspace.info.FileErrorInfo
 import com.intellij.cce.workspace.storages.storage.ActionsSingleFileStorage
+import com.intellij.ide.impl.runUnderModalProgressIfIsEdt
 import com.intellij.openapi.application.ReadAction
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.warmup.util.importOrOpenProjectAsync
+import java.nio.file.FileSystems
 import kotlin.random.Random
 
-open class ProjectActionsDataset(
+open class ProjectActionsEnvironment(
   private val strategy: EvaluationStrategy,
   val config: Config.ActionsGeneration,
   private val filesLimit: Int?, // TODO dataset generation could be lazy
@@ -37,7 +40,7 @@ open class ProjectActionsDataset(
   val processor: GenerateActionsProcessor,
   private val featureName: String,
   val featureInvoker: FeatureInvoker,
-) : EvaluationDataset {
+) : EvaluationEnvironment {
   private val datasetRef = config.sourceFile.run {
     val sf = this ?: ""
     if (sf.isNotBlank()) {
@@ -89,7 +92,7 @@ open class ProjectActionsDataset(
     return datasetContext.actionsStorage.computeSessionsCount()
   }
 
-  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationDatasetChunk> {
+  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationChunk> {
     ensureDataRefIsHandled(datasetContext)
     val files = datasetContext.actionsStorage.getActionFiles()
     return files.shuffled(FILES_RANDOM).asSequence().map { file ->
@@ -170,6 +173,13 @@ open class ProjectActionsDataset(
     actionsSummarizer.save(datasetContext)
   }
 
+  override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? =
+    step.runInIntellij(project, workspace)
+
+  override fun close() {
+    ProjectOpeningUtils.closeProject(project)
+  }
+
   private class ActionsSummarizer {
     private val rootSummary: Summary = Summary.create()
     fun update(fileActions: FileActions) {
@@ -232,7 +242,7 @@ open class ProjectActionsDataset(
   private inner class FileActionsChunk(
     private val fileActions: FileActions,
     override val presentationText: String,
-  ) : EvaluationDatasetChunk {
+  ) : EvaluationChunk {
     override val datasetName: String = config.projectName
     override val name: String = fileActions.path
 
@@ -250,7 +260,28 @@ open class ProjectActionsDataset(
       return actionInterpreter.interpret(fileActions, sessionHandler)
     }
   }
+
+  companion object {
+    fun open(projectPath: String, init: (Project) -> ProjectActionsEnvironment): ProjectActionsEnvironment {
+      println("Open and load project $projectPath. Operation may take a few minutes.")
+      @Suppress("DEPRECATION")
+      val project = runUnderModalProgressIfIsEdt {
+        importOrOpenProjectAsync(OpenProjectArgsData(FileSystems.getDefault().getPath(projectPath)))
+      }
+      println("Project loaded!")
+
+      val environment = try {
+        init(project)
+      }
+      catch (exception: Exception) {
+        ProjectOpeningUtils.closeProject(project)
+        throw RuntimeException("Failed to initialize project environment: $exception", exception)
+      }
+
+      return environment
+    }
+  }
 }
 
-private val LOG = Logger.getInstance(ProjectActionsDataset::class.java)
+private val LOG = Logger.getInstance(ProjectActionsEnvironment::class.java)
 private val FILES_RANDOM = Random(42)

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt

@@ -1,8 +1,7 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.evaluable
 
-import com.intellij.cce.actions.DatasetRef
-import com.intellij.cce.actions.ProjectActionsDataset
+import com.intellij.cce.actions.ProjectActionsEnvironment
 import com.intellij.cce.core.Language
 import com.intellij.cce.evaluation.*
 import com.intellij.cce.interpreter.FeatureInvoker
@@ -44,19 +43,17 @@ abstract class EvaluableFeatureBase<T : EvaluationStrategy>(override val name: S
   override fun prepareEnvironment(config: Config): EvaluationEnvironment {
     val actions = actions(config)
     val strategy = config.strategy<T>()
-    return ProjectEnvironment.open(actions.projectPath) { project ->
-      StandaloneEnvironment(
-        dataset = ProjectActionsDataset(
-          strategy,
-          actions,
-          config.interpret.filesLimit,
-          config.interpret.sessionsLimit,
-          EvaluationRootInfo(true),
-          project,
-          getGenerateActionsProcessor(strategy, project),
-          name,
-          featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy)
-        )
+    return ProjectActionsEnvironment.open(actions.projectPath) { project ->
+      ProjectActionsEnvironment(
+        strategy,
+        actions,
+        config.interpret.filesLimit,
+        config.interpret.sessionsLimit,
+        EvaluationRootInfo(true),
+        project,
+        getGenerateActionsProcessor(strategy, project),
+        name,
+        featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy)
       )
     }
   }

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt

@@ -1,10 +1,6 @@
 package com.intellij.cce.evaluable
 
-import com.intellij.cce.actions.EvaluationDataset
-import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.evaluation.EvaluationStep
-import com.intellij.cce.evaluation.StandaloneEnvironment
-import com.intellij.cce.interpreter.FeatureInvoker
 import com.intellij.cce.report.BasicFileReportGenerator
 import com.intellij.cce.report.FileReportGenerator
 import com.intellij.cce.report.GeneratorDirectories
@@ -16,8 +12,6 @@ abstract class StandaloneFeature<T : EvaluationStrategy>(
   override val name: String
 ) : EvaluableFeature<T> {
 
-  abstract fun getDataset(config: Config): EvaluationDataset
-
   override fun getPreliminaryEvaluationSteps(): List<EvaluationStep> = emptyList()
 
   override fun getEvaluationSteps(config: Config): List<EvaluationStep> = emptyList()
@@ -29,10 +23,4 @@ abstract class StandaloneFeature<T : EvaluationStrategy>(
     fullLineStorages: List<FullLineLogsStorage>,
     dirs: GeneratorDirectories
   ): FileReportGenerator = BasicFileReportGenerator(filterName, comparisonFilterName, featuresStorages, dirs)
-
-  override fun prepareEnvironment(config: Config): EvaluationEnvironment {
-    return StandaloneEnvironment(
-      getDataset(config)
-    )
-  }
 }

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictEnvironment.kt

@@ -3,32 +3,26 @@ package com.intellij.cce.evaluable.conflictResolution
 import com.intellij.cce.actions.*
 import com.intellij.cce.core.*
 import com.intellij.cce.evaluable.AIA_CONTEXT
-import com.intellij.cce.evaluation.EvaluationStep
+import com.intellij.cce.evaluation.EvaluationChunk
+import com.intellij.cce.evaluation.SimpleFileEnvironment
 import com.intellij.cce.interpreter.InterpretFilter
 import com.intellij.cce.interpreter.InterpretationHandler
 import com.intellij.cce.interpreter.InterpretationOrder
-import com.intellij.cce.util.Progress
 import java.nio.file.Path
 import kotlin.io.path.listDirectoryEntries
 import kotlin.io.path.readText
 
-class ConflictDataset(
-  private val datasetRef: DatasetRef,
+class ConflictEnvironment(
+  override val datasetRef: DatasetRef,
   private val conflictResolver: ConflictResolver<*>
-) : EvaluationDataset {
-  override val setupSdk: EvaluationStep? = null
-  override val checkSdk: EvaluationStep? = null
+) : SimpleFileEnvironment() {
 
   override val preparationDescription: String = "Checking that conflict dataset exists"
 
-  override fun prepare(datasetContext: DatasetContext, progress: Progress) {
-    datasetRef.prepare(datasetContext)
-  }
-
   override fun sessionCount(datasetContext: DatasetContext): Int =
     datasetContext.path(datasetRef).listDirectoryEntries().size // FIXME should be pre-calculated
 
-  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationDatasetChunk> {
+  override fun chunks(datasetContext: DatasetContext): Iterator<EvaluationChunk> {
     return datasetContext.path(datasetRef).listDirectoryEntries().map { conflictPath ->
       ConflictChunk(conflictPath)
     }.iterator()
@@ -36,7 +30,7 @@ class ConflictDataset(
 
   private inner class ConflictChunk(
     private val conflictPath: Path,
-  ) : EvaluationDatasetChunk {
+  ) : EvaluationChunk {
     override val datasetName: String = datasetRef.name
     override val name: String = conflictPath.fileName.toString()
     override val presentationText: String = readText("result")

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt

@@ -1,9 +1,9 @@
 package com.intellij.cce.evaluable.conflictResolution
 
 import com.intellij.cce.actions.DatasetRef
-import com.intellij.cce.actions.EvaluationDataset
 import com.intellij.cce.evaluable.StandaloneFeature
 import com.intellij.cce.evaluable.StrategySerializer
+import com.intellij.cce.evaluation.SimpleFileEnvironment
 import com.intellij.cce.metric.Metric
 import com.intellij.cce.metric.PrecisionMetric
 import com.intellij.cce.metric.SessionsCountMetric
@@ -12,7 +12,7 @@ import com.intellij.cce.workspace.Config
 class ConflictResolutionFeature : StandaloneFeature<ConflictResolutionStrategy>("conflict-resolution") {
   override fun getStrategySerializer(): StrategySerializer<ConflictResolutionStrategy> = ConflictResolutionStrategy.Serializer()
 
-  override fun getDataset(config: Config): EvaluationDataset = ConflictDataset(
+  override fun prepareEnvironment(config: Config): SimpleFileEnvironment = ConflictEnvironment(
     DatasetRef.parse(config.conflictDataset!!.url),
     TheirConflictResolver()
   )

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt

@@ -2,12 +2,12 @@ package com.intellij.cce.evaluable.standaloneExample
 
 import com.google.gson.JsonObject
 import com.google.gson.JsonSerializationContext
-import com.intellij.cce.actions.CsvDataset
+import com.intellij.cce.actions.CsvEnvironment
 import com.intellij.cce.actions.DatasetRef
-import com.intellij.cce.actions.EvaluationDataset
 import com.intellij.cce.evaluable.EvaluationStrategy
 import com.intellij.cce.evaluable.StandaloneFeature
 import com.intellij.cce.evaluable.StrategySerializer
+import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.filter.EvaluationFilter
 import com.intellij.cce.metric.Metric
 import com.intellij.cce.metric.PrecisionMetric
@@ -22,9 +22,9 @@ class StandaloneExampleFeature : StandaloneFeature<DatasetStrategy>("standalone-
     override fun deserialize(map: Map<String, Any>, language: String): DatasetStrategy = DatasetStrategy()
   }
 
-  override fun getDataset(config: Config): EvaluationDataset {
+  override fun prepareEnvironment(config: Config): EvaluationEnvironment {
     val fileDataset = config.csvDataset ?: throw IllegalStateException("Required dataset config")
-    return CsvDataset(
+    return CsvEnvironment(
       datasetRef = DatasetRef.parse(fileDataset.url),
       chunkSize = fileDataset.chunkSize,
       targetField = "Type",

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt

@@ -2,9 +2,7 @@
 package com.intellij.cce.evaluation
 
 import com.intellij.cce.actions.DatasetContext
-import com.intellij.cce.actions.EvaluationDataset
 import com.intellij.cce.evaluation.step.SetupStatsCollectorStep
-import com.intellij.cce.interpreter.FeatureInvoker
 import com.intellij.cce.interpreter.InterpretFilter
 import com.intellij.cce.interpreter.InterpretationHandlerImpl
 import com.intellij.cce.util.ExceptionsUtil
@@ -35,10 +33,10 @@ class ActionsInterpretationHandler(
     logsSaverIf(config.interpret.saveFusLogs) { workspace.fusLogsSaver }
   ).asCompositeLogsSaver()
 
-  fun invoke(dataset: EvaluationDataset, workspace: EvaluationWorkspace, indicator: Progress) {
+  fun invoke(environment: EvaluationEnvironment, workspace: EvaluationWorkspace, indicator: Progress) {
     var sessionsCount: Int
     val computingTime = measureTimeMillis {
-      sessionsCount = dataset.sessionCount(datasetContext)
+      sessionsCount = environment.sessionCount(datasetContext)
     }
     LOG.info("Computing of sessions count took $computingTime ms")
     val interpretationConfig = config.interpret
@@ -55,7 +53,7 @@ class ActionsInterpretationHandler(
       println("During actions interpretation will be skipped about $skippedSessions sessions")
     }
     var fileCount = 0
-    for (chunk in dataset.chunks(datasetContext)) {
+    for (chunk in environment.chunks(datasetContext)) {
       if (config.interpret.filesLimit?.let { it <= fileCount } == true) {
         break
       }

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt

@@ -17,17 +17,17 @@ class BackgroundStepFactory(
   private val datasetContext: DatasetContext
 ) : StepFactory {
 
-  override fun generateActionsStep(): EvaluationStep = DatasetPreparationStep(environment.dataset, datasetContext)
+  override fun generateActionsStep(): EvaluationStep = DatasetPreparationStep(environment, datasetContext)
 
   override fun interpretActionsStep(): EvaluationStep =
-    ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = false)
+    ActionsInterpretationStep(config, environment, datasetContext, newWorkspace = false)
 
   override fun generateReportStep(): EvaluationStep =
     ReportGenerationStep(inputWorkspacePaths?.map { EvaluationWorkspace.open(it, SetupStatsCollectorStep.statsCollectorLogsDirectory) },
                          config.reports.sessionsFilters, config.reports.comparisonFilters, feature)
 
   override fun interpretActionsOnNewWorkspaceStep(): EvaluationStep =
-    ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = true)
+    ActionsInterpretationStep(config, environment, datasetContext, newWorkspace = true)
 
   override fun reorderElements(): EvaluationStep =
     ReorderElementsStep(config)
@@ -39,9 +39,9 @@ class BackgroundStepFactory(
       SetupStatsCollectorStep(config.interpret.experimentGroup, config.interpret.logLocationAndItemText)
     else null
 
-  override fun setupSdkStep(): EvaluationStep? = environment.dataset.setupSdk
+  override fun setupSdkStep(): EvaluationStep? = environment.setupSdk
 
-  override fun checkSdkConfiguredStep(): EvaluationStep? = environment.dataset.checkSdk
+  override fun checkSdkConfiguredStep(): EvaluationStep? = environment.checkSdk
 
   override fun finishEvaluationStep(): FinishEvaluationStep = HeadlessFinishEvaluationStep()
 

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationChunk.kt

@@ -0,0 +1,22 @@
+package com.intellij.cce.evaluation
+
+import com.intellij.cce.core.Session
+import com.intellij.cce.interpreter.InterpretFilter
+import com.intellij.cce.interpreter.InterpretationHandler
+import com.intellij.cce.interpreter.InterpretationOrder
+
+/**
+ * Represents a part of an evaluation which can be handled and represented separately.
+ */
+interface EvaluationChunk {
+  val datasetName: String
+  val name: String
+  val presentationText: String
+
+  fun evaluate(
+    handler: InterpretationHandler,
+    filter: InterpretFilter,
+    order: InterpretationOrder,
+    sessionHandler: (Session) -> Unit
+  ): List<Session>
+}

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt

@@ -1,27 +1,50 @@
 package com.intellij.cce.evaluation
 
-import com.intellij.cce.actions.EvaluationDataset
+import com.intellij.cce.actions.DatasetContext
+import com.intellij.cce.actions.DatasetRef
 import com.intellij.cce.evaluation.step.runInIntellij
-import com.intellij.cce.interpreter.FeatureInvoker
+import com.intellij.cce.util.Progress
 import com.intellij.cce.workspace.EvaluationWorkspace
+import java.nio.file.Path
 
 /**
  * Environment represents resources needed for an evaluation.
- * For example, it can be IntelliJ project like in [ProjectEnvironment] if an evaluation needs an opened project.
+ * For example, it can be IntelliJ project like in ProjectActionsEnvironment if an evaluation needs an opened project.
  * It should be initialized before an evaluation process and closed right after finish.
  */
 interface EvaluationEnvironment : AutoCloseable {
-  val dataset: EvaluationDataset
+
+  val setupSdk: EvaluationStep?
+  val checkSdk: EvaluationStep?
+
+  val preparationDescription: String
+
+  fun prepare(datasetContext: DatasetContext, progress: Progress)
+
+  fun sessionCount(datasetContext: DatasetContext): Int
+
+  // TODO should return something closeable for large files
+  fun chunks(datasetContext: DatasetContext): Iterator<EvaluationChunk>
 
   fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace?
 }
 
-/**
- * A special type of environment which doesn't imply any associated resources and can be treated like a simple data class.
- */
-class StandaloneEnvironment(
-  override val dataset: EvaluationDataset,
-) : EvaluationEnvironment {
+abstract class SimpleFileEnvironment : EvaluationEnvironment {
+
+  protected abstract val datasetRef: DatasetRef
+
+  override val setupSdk: EvaluationStep? = null
+  override val checkSdk: EvaluationStep? = null
+
+  protected open fun checkFile(datasetPath: Path) {
+  }
+
+  override fun prepare(datasetContext: DatasetContext, progress: Progress) {
+    datasetRef.prepare(datasetContext)
+    val datasetPath = datasetContext.path(datasetRef)
+    checkFile(datasetPath)
+  }
+
   override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? =
     step.runInIntellij(null, workspace)
 

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt

@@ -1,49 +0,0 @@
-package com.intellij.cce.evaluation
-
-import com.intellij.cce.actions.EvaluationDataset
-import com.intellij.cce.actions.OpenProjectArgsData
-import com.intellij.cce.actions.ProjectOpeningUtils
-import com.intellij.cce.evaluation.step.runInIntellij
-import com.intellij.cce.interpreter.FeatureInvoker
-import com.intellij.cce.workspace.EvaluationWorkspace
-import com.intellij.ide.impl.runUnderModalProgressIfIsEdt
-import com.intellij.openapi.project.Project
-import com.intellij.warmup.util.importOrOpenProjectAsync
-import java.nio.file.FileSystems
-
-class ProjectEnvironment(
-  val project: Project,
-  override val dataset: EvaluationDataset,
-) : EvaluationEnvironment {
-
-  override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? =
-    step.runInIntellij(project, workspace)
-
-  override fun close() {
-    ProjectOpeningUtils.closeProject(project)
-  }
-
-  companion object {
-    fun open(projectPath: String, init: (Project) -> StandaloneEnvironment): ProjectEnvironment {
-      println("Open and load project $projectPath. Operation may take a few minutes.")
-      @Suppress("DEPRECATION")
-      val project = runUnderModalProgressIfIsEdt {
-        importOrOpenProjectAsync(OpenProjectArgsData(FileSystems.getDefault().getPath(projectPath)))
-      }
-      println("Project loaded!")
-
-      val environment = try {
-        init(project)
-      }
-      catch (exception: Exception) {
-        ProjectOpeningUtils.closeProject(project)
-        throw RuntimeException("Failed to initialize project environment: $exception", exception)
-      }
-
-      return ProjectEnvironment(
-        project,
-        environment.dataset
-      )
-    }
-  }
-}

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt

@@ -2,16 +2,15 @@
 package com.intellij.cce.evaluation.step
 
 import com.intellij.cce.actions.DatasetContext
-import com.intellij.cce.actions.EvaluationDataset
 import com.intellij.cce.evaluation.ActionsInterpretationHandler
-import com.intellij.cce.interpreter.FeatureInvoker
+import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.util.Progress
 import com.intellij.cce.workspace.Config
 import com.intellij.cce.workspace.EvaluationWorkspace
 
 class ActionsInterpretationStep(
   private val config: Config,
-  private val dataset: EvaluationDataset,
+  private val environment: EvaluationEnvironment,
   private val datasetContext: DatasetContext,
   private val newWorkspace: Boolean
 ) : BackgroundEvaluationStep {
@@ -23,7 +22,7 @@ class ActionsInterpretationStep(
     val resultWorkspace =
       if (newWorkspace) EvaluationWorkspace.create(config, SetupStatsCollectorStep.statsCollectorLogsDirectory)
       else workspace
-    ActionsInterpretationHandler(config, datasetContext).invoke(dataset, resultWorkspace, progress)
+    ActionsInterpretationHandler(config, datasetContext).invoke(environment, resultWorkspace, progress)
     return resultWorkspace
   }
 }

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/DatasetPreparationStep.kt

@@ -1,20 +1,20 @@
 package com.intellij.cce.evaluation.step
 
 import com.intellij.cce.actions.DatasetContext
-import com.intellij.cce.actions.EvaluationDataset
+import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.util.Progress
 import com.intellij.cce.workspace.EvaluationWorkspace
 
 class DatasetPreparationStep(
-  private val dataset: EvaluationDataset,
+  private val environment: EvaluationEnvironment,
   private val datasetContext: DatasetContext,
 ) : BackgroundEvaluationStep {
   override val name: String = "Preparing dataset"
 
-  override val description: String = dataset.preparationDescription
+  override val description: String = environment.preparationDescription
 
   override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace {
-    dataset.prepare(datasetContext, progress)
+    environment.prepare(datasetContext, progress)
     return workspace
   }
 }