[evaluation-plugin] LME-377 Refactor sdk setup mechanism

- Configure interpreter in a more independent way and resolve deps for python environment
- Use the new mechanism for test-generation evaluation


Merge-request: IJ-MR-159820
Merged-by: Roman Vasiliev <Roman.Vasiliev@jetbrains.com>

GitOrigin-RevId: ab945d70a5e9a9c861c5bc94437c708479c84b6f

plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/EvaluationStrategy.kt

@@ -1,16 +1,12 @@
 // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.evaluable
 
-import com.intellij.cce.evaluation.data.ExecutionMode
 import com.intellij.cce.filter.EvaluationFilter
 
 interface EvaluationStrategy {
 
   val filters: Map<String, EvaluationFilter>
 
-  val executionMode: ExecutionMode
-    get() = ExecutionMode.LOCAL
-
   companion object {
     val defaultStrategy = object : EvaluationStrategy {
       override val filters: Map<String, EvaluationFilter> = mapOf()

plugins/evaluation-plugin/languages/python/resources/intellij.evaluationPlugin.languages.python.xml

@@ -8,7 +8,7 @@
     <completionEvaluationVisitor implementation="com.intellij.cce.python.visitor.PythonMultiLineEvaluationVisitor"/>
     <completionEvaluationVisitor implementation="com.intellij.cce.python.visitor.PythonTextCompletionEvaluationVisitor"/>
     <lineCompletionVisitorFactory implementation="com.intellij.cce.python.visitor.PythonLineCompletionVisitorFactory"/>
-    <setupSdkStep implementation="com.intellij.cce.python.evaluation.SetupPythonInterpreterStep"/>
+    <setupSdkStep implementation="com.intellij.cce.python.evaluation.SetupPythonInterpreterStepFactory"/>
     <completionEvaluationVisitor implementation="com.intellij.cce.python.visitor.PythonCompletionContextEvaluationVisitor"/>
     <completionEvaluationVisitor implementation="com.intellij.cce.python.visitor.PythonDocGenerationVisitor"/>
     <completionEvaluationVisitor implementation="com.intellij.cce.python.visitor.PythonCodeGenerationVisitor"/>

plugins/evaluation-plugin/languages/python/src/com/intellij/cce/python/evaluation/SetupPythonInterpreterStep.kt

@@ -2,37 +2,104 @@
 package com.intellij.cce.python.evaluation
 
 import com.intellij.cce.core.Language
-import com.intellij.cce.evaluable.EvaluationStrategy
-import com.intellij.cce.evaluation.SetupSdkStep
-import com.intellij.cce.evaluation.data.ExecutionMode
+import com.intellij.cce.evaluation.EvaluationStep
+import com.intellij.cce.evaluation.ForegroundEvaluationStep
+import com.intellij.cce.evaluation.SetupSdkPreferences
+import com.intellij.cce.evaluation.SetupSdkStepFactory
 import com.intellij.cce.workspace.EvaluationWorkspace
 import com.intellij.ide.plugins.PluginManagerCore
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.WriteAction
 import com.intellij.openapi.extensions.PluginId
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.progress.runBlockingCancellable
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.projectRoots.ProjectJdkTable
+import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
 import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
+import com.jetbrains.python.packaging.common.PythonPackage
+import com.jetbrains.python.packaging.common.PythonPackageSpecificationBase
+import com.jetbrains.python.packaging.management.PythonPackageManager
+import com.jetbrains.python.packaging.pip.PipPythonPackageManager
+import com.jetbrains.python.packaging.requirement.PyRequirementRelation
 import com.jetbrains.python.sdk.PythonSdkType
+import com.jetbrains.python.sdk.configuration.PyProjectSdkConfiguration
+import java.io.IOException
+import java.nio.file.Path
+import kotlin.io.path.Path
+import kotlin.io.path.exists
 
-class SetupPythonInterpreterStep(private val project: Project) : SetupSdkStep() {
+class SetupPythonInterpreterStepFactory(private val project: Project) : SetupSdkStepFactory {
+  override fun isApplicable(language: Language): Boolean = language == Language.PYTHON
+  override fun steps(preferences: SetupSdkPreferences): List<EvaluationStep> = listOf(
+    SetupPythonInterpreterStep(project, preferences)
+  )
+}
+
+private class SetupPythonInterpreterStep(
+  private val project: Project,
+  private val preferences: SetupSdkPreferences
+) : ForegroundEvaluationStep {
   companion object {
     private const val pythonPluginId = "PythonCore"
     private const val pythonPluginProId = "Pythonid"
   }
 
   override val name: String = "Set up Python Interpreter step"
-  override val description: String = "Configure project Python Interpreter if needed"
-
-  override fun isApplicable(language: Language): Boolean = language == Language.PYTHON
-
-  override fun isApplicable(language: Language, strategy: EvaluationStrategy): Boolean =
-    strategy.executionMode == ExecutionMode.LOCAL && isApplicable(language)
+  override val description: String = "Configure project Python Interpreter and install deps from requirements.txt"
 
   override fun start(workspace: EvaluationWorkspace): EvaluationWorkspace? {
+    val projectSdk = ProjectRootManager.getInstance(project).projectSdk
+
+    val sdk = if (projectSdk != null && (!preferences.projectLocal || isProjectLocal(projectSdk.homePath))) {
+      println("Project SDK already configured")
+      projectSdk
+    }
+    else {
+      println("Project SDK not configured")
+      configureSdk()
+    }
+
+    if (sdk == null) {
+      return null
+    }
+
+    if (preferences.resolveDeps) {
+      runBlockingCancellable {
+        installPackages(sdk)
+      }
+    }
+
+    return workspace
+  }
+
+  private fun configureSdk(): Sdk? {
+    val projectLocalVenvDir = Path.of(project.basePath ?: error("Project path is not found")).resolve(".venv")
+    val existingSdkPath = providedSdkPath() ?: projectLocalVenvSdkPath(projectLocalVenvDir)
+
+    val sdkPath =
+      if (existingSdkPath == null) systemSdkPath()?.let { initProjectLocalVenv(it, projectLocalVenvDir) }
+      else {
+        if (preferences.projectLocal) {
+          if (isProjectLocal(existingSdkPath)) existingSdkPath
+          else initProjectLocalVenv(existingSdkPath, projectLocalVenvDir)
+        }
+        else existingSdkPath
+      }
+
+
+    if (sdkPath == null) {
+      println("Project SDK path was not provided. Setup `EVALUATION_PYTHON` or `PYTHONPATH` env variable")
+      return null
+    }
+
+    return getSdk(sdkPath)
+  }
+
+  private fun getSdk(sdkHomePath: String): Sdk? {
     val pythonPluginEnabled = PluginManagerCore.getPlugin(PluginId.getId(pythonPluginId))?.isEnabled ?: false
     val pythonPluginProEnabled = PluginManagerCore.getPlugin(PluginId.getId(pythonPluginProId))?.isEnabled ?: false
     if (!pythonPluginEnabled && !pythonPluginProEnabled) {
@@ -40,39 +107,138 @@ class SetupPythonInterpreterStep(private val project: Project) : SetupSdkStep()
       return null
     }
 
+    var resultSdk: Sdk? = null
     ApplicationManager.getApplication().invokeAndWait {
-      val projectRootManager = ProjectRootManager.getInstance(project)
-      val projectSdk = projectRootManager.projectSdk
-      if (projectSdk != null) {
-        println("Project SDK already configured")
+      val sdkHome = WriteAction.compute<VirtualFile, RuntimeException> {
+        LocalFileSystem.getInstance().refreshAndFindFileByPath(sdkHomePath)
       }
-      else {
-        println("Project SDK not configured")
-        val sdkHomePath = System.getenv("EVALUATION_PYTHON") ?: System.getenv("PYTHONPATH")
-        if (sdkHomePath == null) {
-          println("Project SDK path was not provided. Setup `EVALUATION_PYTHON` or `PYTHONPATH` env variable")
-        }
-        val sdkHome = WriteAction.compute<VirtualFile, RuntimeException> {
-          LocalFileSystem.getInstance().refreshAndFindFileByPath(sdkHomePath)
-        }
-        if (sdkHome == null) {
-          println("Failed to find SDK home directory at path: $sdkHomePath")
-          return@invokeAndWait
-        }
-        val sdk = SdkConfigurationUtil.setupSdk(emptyArray(), sdkHome, PythonSdkType.getInstance(), true, null, sdkHome.path)
-        if (sdk != null) {
-          WriteAction.run<Throwable> {
-            val sdkTable = ProjectJdkTable.getInstance()
-            if (sdkTable.findJdk(sdk.name) == null) {
-              sdkTable.addJdk(sdk)
+      if (sdkHome == null) {
+        println("Failed to find SDK home directory at path: $sdkHomePath")
+        return@invokeAndWait
+      }
+      val sdk = SdkConfigurationUtil.setupSdk(emptyArray(), sdkHome, PythonSdkType.getInstance(), true, null, sdkHome.path)
+      if (sdk != null) {
+        WriteAction.run<Throwable> {
+          val sdkTable = ProjectJdkTable.getInstance()
+          val existingSdk = sdkTable.findJdk(sdk.name)
+          if (existingSdk?.homePath != sdk.homePath) {
+            if (existingSdk != null) {
+              sdkTable.removeJdk(existingSdk)
             }
-            projectRootManager.projectSdk = sdk
+            sdkTable.addJdk(sdk)
+          }
+          for (module in ModuleManager.getInstance(project).modules) {
+            PyProjectSdkConfiguration.setReadyToUseSdk(project, module, sdk)
           }
-          println("Python interpreter \"${sdk.name}\" (${sdk.homePath}) will be used as a project SDK")
         }
+        println("Python interpreter \"${sdk.name}\" (${sdk.homePath}) will be used as a project SDK")
+        resultSdk = sdk
       }
     }
+    return resultSdk
+  }
 
-    return workspace
+  private suspend fun installPackages(sdk: Sdk) {
+    val packageManager = PythonPackageManager.forSdk(project, sdk)
+    packageManager.reloadPackages()
+
+    val packages = readRequiredPackages().filterNot { packageManager.packageExists(PythonPackage(it.name, "", false)) }
+    if (packages.isEmpty()) {
+      println("No packages to install. Skipping.")
+      return
+    }
+
+    val cacheOptions = if (preferences.cacheDir == null) emptyList() else when (packageManager) {
+      is PipPythonPackageManager -> listOf("--cache-dir=${preferences.cacheDir}/pip")
+      else -> emptyList()
+    }
+
+    // resolves `'runBlockingCancellable' is forbidden in the Write Action` from PythonSdkUpdater.scheduleUpdate
+    keepTasksAsynchronousInHeadlessMode {
+      packageManager.installPackages(packages, cacheOptions, false)
+      println("Installed packages: ${packages.joinToString(", ") {it.name}}")
+    }
+  }
+
+  private fun readRequiredPackages(): List<PythonPackageSpecificationBase> {
+    val projectPath = project.basePath ?: return emptyList()
+    val requirementsTxt = Path.of(projectPath).resolve("requirements.txt")
+    if (!requirementsTxt.exists()) {
+      return emptyList()
+    }
+    return requirementsTxt.toFile().inputStream().bufferedReader().use {
+      it.readLines().filter { it.isNotBlank() }.map { line ->
+        val relation = PyRequirementRelation.entries.find { line.contains(it.presentableText) }
+        val parts = if (relation != null) line.split(relation.presentableText).map { it.trim() } else listOf(line.trim())
+        PythonPackageSpecificationBase(parts[0], parts.getOrNull(1), relation, null)
+      }
+    }
+  }
+
+  private fun isProjectLocal(path: String?): Boolean {
+    if (path == null) return false
+    val projectDir = project.basePath ?: return false
+    return Path(path).startsWith(Path(projectDir))
   }
 }
+
+private fun providedSdkPath(): String? {
+  val provided = System.getenv("EVALUATION_PYTHON") ?: System.getenv("PYTHONPATH")
+  return if (provided?.isNotBlank() == true) provided else null
+}
+
+private fun systemSdkPath(): String? {
+  val home = "/usr/bin/python3"
+
+  try {
+    val process = ProcessBuilder(home, "--version")
+      .redirectOutput(ProcessBuilder.Redirect.DISCARD)
+      .redirectError(ProcessBuilder.Redirect.DISCARD)
+      .start()
+
+    process.waitFor()
+
+    return home.takeIf { process.exitValue() == 0 }
+  }
+  catch (_: IOException) {
+    return null
+  }
+}
+
+private fun projectLocalVenvSdkPath(venvDir: Path): String? {
+  if (!venvDir.exists()) return null
+  return venvDir.resolve("bin/python").toAbsolutePath().toString()
+}
+
+private fun initProjectLocalVenv(parentSdkPath: String, venvDir: Path): String? {
+  println("Creating virtual environment in $venvDir")
+
+  val process = ProcessBuilder(parentSdkPath, "-m", "venv", venvDir.fileName.toString())
+    .directory(venvDir.parent.toFile())
+    .redirectOutput(ProcessBuilder.Redirect.INHERIT)
+    .redirectError(ProcessBuilder.Redirect.INHERIT)
+    .start()
+
+  process.waitFor()
+
+  if (process.exitValue() != 0) {
+    println("Failed to create virtual environment in $venvDir/.venv")
+    return null
+  }
+
+  return projectLocalVenvSdkPath(venvDir)
+}
+
+private suspend fun <T> keepTasksAsynchronousInHeadlessMode(f: suspend () -> T): T {
+  val propertyName = "intellij.progress.task.ignoreHeadless"
+  val previousValue = System.getProperty(propertyName)
+  try {
+    System.setProperty(propertyName, "true")
+    return f()
+  }
+  finally {
+    previousValue?.let {
+      System.setProperty(propertyName, it)
+    }
+  }
+}

plugins/evaluation-plugin/languages/src/com/intellij/cce/evaluation/SetupSdkStep.kt

@@ -2,23 +2,35 @@
 package com.intellij.cce.evaluation
 
 import com.intellij.cce.core.Language
-import com.intellij.cce.evaluation.data.ExecutionMode
-import com.intellij.cce.evaluable.EvaluationStrategy
 import com.intellij.openapi.extensions.ExtensionPointName
 import com.intellij.openapi.project.Project
 
-abstract class SetupSdkStep : ForegroundEvaluationStep {
+interface SetupSdkStepFactory {
   companion object {
-    private val EP_NAME = ExtensionPointName.create<SetupSdkStep>("com.intellij.cce.setupSdkStep")
-    fun forLanguage(project: Project, language: Language, strategy: EvaluationStrategy): SetupSdkStep? {
+    private val EP_NAME = ExtensionPointName.create<SetupSdkStepFactory>("com.intellij.cce.setupSdkStep")
+    fun forLanguage(project: Project, language: Language): SetupSdkStepFactory? {
       return EP_NAME.getExtensionList(project).firstOrNull {
-        it.isApplicable(language, strategy)
+        it.isApplicable(language)
       }
     }
   }
 
-  abstract fun isApplicable(language: Language): Boolean
+  fun isApplicable(language: Language): Boolean
 
-  open fun isApplicable(language: Language, strategy: EvaluationStrategy): Boolean =
-    strategy.executionMode == ExecutionMode.LOCAL && isApplicable(language)
+  fun steps(preferences: SetupSdkPreferences): List<EvaluationStep>
 }
+
+abstract class SetupSdkStep : SetupSdkStepFactory, ForegroundEvaluationStep {
+  override fun steps(preferences: SetupSdkPreferences): List<EvaluationStep> = listOf(this)
+}
+
+/**
+ * @property resolveDeps Indicates whether setup should try to resolve dependencies
+ * @property projectLocal Should prefer project-local installation if applicable (i.e. use local .venv for python)
+ * @property cacheDir Path to a directory which can be used by setup steps to store caches. No need to make anything with caches if null
+ */
+data class SetupSdkPreferences(
+  val resolveDeps: Boolean,
+  val projectLocal: Boolean = resolveDeps,
+  val cacheDir: String? = null
+)

plugins/evaluation-plugin/languages/src/com/intellij/cce/execution/manager/CodeExecutionManager.kt

@@ -26,7 +26,6 @@ abstract class CodeExecutionManager {
 
   abstract val language: Language
   abstract val executionMode: ExecutionMode?
-  var shouldSetup: Boolean = true
 
   private val executionBasedMetrics = listOf(AIA_EXECUTION_SUCCESS_RATIO, AIA_TEST_LINE_COVERAGE, AIA_TEST_BRANCH_COVERAGE, AIA_TEST_FILE_PROVIDED)
 
@@ -59,8 +58,6 @@ abstract class CodeExecutionManager {
     val codeFile = getGeneratedCodeFile(basePath, code)
     // Save code in a temp file
     codeFile.writeText(code)
-    // Setup environment
-    setupEnvironment(project, sdk)
     // Compile
     val compilationExecutionLog = compileGeneratedCode()
     if (compilationExecutionLog.exitCode != 0) return compilationExecutionLog

plugins/evaluation-plugin/resources/META-INF/plugin.xml

@@ -58,7 +58,7 @@
                     interface="com.intellij.cce.actions.OpenProjectMethodProvider"
                     dynamic="true"/>
     <extensionPoint qualifiedName="com.intellij.cce.setupSdkStep" area="IDEA_PROJECT"
-                    interface="com.intellij.cce.evaluation.SetupSdkStep"
+                    interface="com.intellij.cce.evaluation.SetupSdkStepFactory"
                     dynamic="true"/>
     <extensionPoint qualifiedName="com.intellij.cce.evaluableFeature"
                     interface="com.intellij.cce.evaluable.EvaluableFeature"

plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt

@@ -69,6 +69,7 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter.
         evaluationRootInfo,
         environment.project,
         environment.processor,
+        environment.setupSteps,
         feature.name,
         environment.featureInvoker
       ) {

plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsEnvironment.kt

@@ -5,7 +5,6 @@ import com.intellij.cce.core.*
 import com.intellij.cce.evaluable.EvaluationStrategy
 import com.intellij.cce.evaluable.common.CommonActionsInvoker
 import com.intellij.cce.evaluation.*
-import com.intellij.cce.evaluation.step.CheckProjectSdkStep
 import com.intellij.cce.evaluation.step.runInIntellij
 import com.intellij.cce.interpreter.*
 import com.intellij.cce.processor.DefaultEvaluationRootProcessor
@@ -41,6 +40,7 @@ open class ProjectActionsEnvironment(
   private val evaluationRootInfo: EvaluationRootInfo,
   val project: Project,
   val processor: GenerateActionsProcessor,
+  override val setupSteps: List<EvaluationStep>,
   private val featureName: String,
   val featureInvoker: FeatureInvoker,
 ) : EvaluationEnvironment {
@@ -54,9 +54,6 @@ open class ProjectActionsEnvironment(
   }
   private var datasetRefIsHandled = false
 
-  override val setupSdk: EvaluationStep? = SetupSdkStep.forLanguage(project, Language.resolve(config.language), strategy)
-  override val checkSdk: EvaluationStep? = CheckProjectSdkStep(project, config.language)
-
   override val preparationDescription: String = "Generating actions by selected files"
 
   override fun prepare(datasetContext: DatasetContext, progress: Progress) {

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt

@@ -6,6 +6,9 @@ import com.intellij.cce.core.Language
 import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.evaluation.EvaluationRootInfo
 import com.intellij.cce.evaluation.EvaluationStep
+import com.intellij.cce.evaluation.SetupSdkPreferences
+import com.intellij.cce.evaluation.SetupSdkStepFactory
+import com.intellij.cce.evaluation.step.CheckProjectSdkStep
 import com.intellij.cce.interpreter.FeatureInvoker
 import com.intellij.cce.processor.GenerateActionsProcessor
 import com.intellij.cce.report.BasicFileReportGenerator
@@ -16,8 +19,12 @@ import com.intellij.cce.workspace.EvaluationWorkspace
 import com.intellij.cce.workspace.storages.FeaturesStorage
 import com.intellij.cce.workspace.storages.FullLineLogsStorage
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.registry.Registry
 
 abstract class EvaluableFeatureBase<T : EvaluationStrategy>(override val name: String) : EvaluableFeature<T> {
+  open val setupSdkPreferences: SetupSdkPreferences = SetupSdkPreferences(
+    resolveDeps = false
+  )
 
   /**
    * how to prepare the context before the feature invocation
@@ -34,6 +41,9 @@ abstract class EvaluableFeatureBase<T : EvaluationStrategy>(override val name: S
   override fun getEvaluationSteps(config: Config): List<EvaluationStep> =
     getEvaluationSteps(Language.resolve(actions(config).language), config.strategy())
 
+  open fun getSetupSteps(project: Project, language: Language, strategy: T): List<EvaluationStep> =
+    defaultSetupSteps(project, language, setupSdkPreferences)
+
   open fun getFileReportGenerator(
     filterName: String,
     comparisonFilterName: String,
@@ -68,6 +78,7 @@ abstract class EvaluableFeatureBase<T : EvaluationStrategy>(override val name: S
         EvaluationRootInfo(true),
         project,
         getGenerateActionsProcessor(strategy, project),
+        getSetupSteps(project, Language.resolve(actions.language), strategy),
         name,
         featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy)
       )
@@ -77,3 +88,9 @@ abstract class EvaluableFeatureBase<T : EvaluationStrategy>(override val name: S
   private fun actions(config: Config) =
     config.actions ?: throw IllegalStateException("Configuration missing project description (actions)")
 }
+
+internal fun defaultSetupSteps(project: Project, language: Language, preferences: SetupSdkPreferences): List<EvaluationStep> {
+  val setupSteps = SetupSdkStepFactory.forLanguage(project, language)?.steps(preferences) ?: emptyList()
+  val checkStep = CheckProjectSdkStep(project, language.displayName).takeUnless { Registry.`is`("evaluation.plugin.disable.sdk.check") }
+  return setupSteps + listOfNotNull(checkStep)
+}

plugins/evaluation-plugin/src/com/intellij/cce/evaluable/PresentableEvaluableFeatureBase.kt

@@ -1,10 +1,13 @@
 package com.intellij.cce.evaluable
 
 import com.intellij.cce.actions.ProjectActionsEnvironment
+import com.intellij.cce.core.Language
 import com.intellij.cce.core.Session
 import com.intellij.cce.core.TokenProperties
 import com.intellij.cce.evaluation.EvaluationEnvironment
 import com.intellij.cce.evaluation.EvaluationRootInfo
+import com.intellij.cce.evaluation.EvaluationStep
+import com.intellij.cce.evaluation.SetupSdkPreferences
 import com.intellij.cce.evaluation.StopEvaluationException
 import com.intellij.cce.interpreter.FeatureInvoker
 import com.intellij.cce.interpreter.PresentableFeatureInvoker
@@ -18,6 +21,10 @@ import com.intellij.openapi.project.Project
  * allowing to use easily customizable evaluation report format for project-based evaluations.
  */
 abstract class PresentableEvaluableFeatureBase<T : EvaluationStrategy>(name: String) : PresentableFeature<T>(name) {
+  open val setupSdkPreferences: SetupSdkPreferences = SetupSdkPreferences(
+    resolveDeps = false
+  )
+
   /**
    * how to prepare the context before the feature invocation
    */
@@ -25,6 +32,9 @@ abstract class PresentableEvaluableFeatureBase<T : EvaluationStrategy>(name: Str
 
   abstract fun getFeatureInvoker(project: Project, strategy: T): PresentableFeatureInvoker
 
+  open fun getSetupSteps(project: Project, language: Language, strategy: T): List<EvaluationStep> =
+    defaultSetupSteps(project, language, setupSdkPreferences)
+
   override fun prepareEnvironment(config: Config, outputWorkspace: EvaluationWorkspace): EvaluationEnvironment {
     val actions = actions(config)
     val strategy = config.strategy<T>()
@@ -37,6 +47,7 @@ abstract class PresentableEvaluableFeatureBase<T : EvaluationStrategy>(name: Str
         EvaluationRootInfo(true),
         project,
         getGenerateActionsProcessor(strategy, project),
+        getSetupSteps(project, Language.resolve(actions.language), strategy),
         name,
         featureInvoker = CustomizableFeatureWrapper(getFeatureInvoker(project, strategy), layoutManager(outputWorkspace))
       )

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt

@@ -41,9 +41,7 @@ class BackgroundStepFactory(
 
   override fun setupRegistryStep(): EvaluationStep = SetupRegistryStep(config.interpret.registry)
 
-  override fun setupSdkStep(): EvaluationStep? = environment.setupSdk
-
-  override fun checkSdkConfiguredStep(): EvaluationStep? = environment.checkSdk
+  override fun setupEnvironmentSteps(): List<EvaluationStep> = environment.setupSteps
 
   override fun finishEvaluationStep(): FinishEvaluationStep = HeadlessFinishEvaluationStep()
 

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt

@@ -16,8 +16,7 @@ import java.nio.file.Path
  */
 interface EvaluationEnvironment : AutoCloseable {
 
-  val setupSdk: EvaluationStep?
-  val checkSdk: EvaluationStep?
+  val setupSteps: List<EvaluationStep>
 
   val preparationDescription: String
 
@@ -38,8 +37,7 @@ interface SimpleFileEnvironment : EvaluationEnvironment {
 
   val datasetRef: DatasetRef
 
-  override val setupSdk: EvaluationStep? get() = null
-  override val checkSdk: EvaluationStep? get() = null
+  override val setupSteps: List<EvaluationStep> get() = emptyList()
 
   override val preparationDescription: String get() = "Checking that dataset file is available"
 

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationProcess.kt

@@ -3,7 +3,6 @@ package com.intellij.cce.evaluation
 
 import com.intellij.cce.workspace.EvaluationWorkspace
 import com.intellij.openapi.application.ApplicationManager
-import com.intellij.openapi.util.registry.Registry
 import kotlin.system.measureTimeMillis
 
 class EvaluationProcess private constructor (
@@ -52,13 +51,7 @@ class EvaluationProcess private constructor (
       val isTestingEnvironment = ApplicationManager.getApplication().isUnitTestMode
 
       if (!isTestingEnvironment && (shouldGenerateActions || shouldInterpretActions)) {
-        factory.setupSdkStep()?.let { steps.add(it) }
-
-        if (!Registry.`is`("evaluation.plugin.disable.sdk.check")) {
-          factory.checkSdkConfiguredStep()?.let {
-            steps.add(it)
-          }
-        }
+        factory.setupEnvironmentSteps().forEach { steps.add(it) }
       }
 
       if (shouldInterpretActions) {

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/StepFactory.kt

@@ -9,8 +9,7 @@ interface StepFactory {
   fun generateReportStep(): EvaluationStep
   fun setupStatsCollectorStep(): EvaluationStep?
   fun setupRegistryStep(): EvaluationStep
-  fun setupSdkStep(): EvaluationStep?
-  fun checkSdkConfiguredStep(): EvaluationStep?
+  fun setupEnvironmentSteps(): List<EvaluationStep>
   fun finishEvaluationStep(): FinishEvaluationStep
   fun featureSpecificSteps(): List<EvaluationStep>
   fun featureSpecificPreliminarySteps(): List<EvaluationStep>