[evaluation-plugin] ML-2550 Option to exclude a folder from eval

GitOrigin-RevId: a303ab8b4ee385dc08fa9e64e689fd1086da33e8

plugins/evaluation-plugin/README.md

@@ -107,6 +107,7 @@ Here is an example of such file with description for possible options but the st
   },
   "actions": { // part of config about actions generation step
     "evaluationRoots": [], // list of string with paths to files/directories for evaluation
+    "ignoreFileNames": [] // list of file/directory names to be ignored inside evaluationRoots
   },
   "interpret": { // part of config about actions interpretation step
     "sessionProbability": 1.0, // probability that session won't be skipped

plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt

@@ -1,4 +1,4 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.workspace
 
 import com.intellij.cce.evaluable.EvaluationStrategy
@@ -10,6 +10,20 @@ import com.intellij.cce.workspace.filter.SessionsFilter
 import java.nio.file.Paths
 import kotlin.io.path.absolute
 
+
+/**
+ * Represents a configuration of the evaluation process.
+ *
+ * @property projectPath The path to the project that will be used for the evaluation.
+ * @property projectName The name of the project. It may differ from the root directory name.
+ * @property language The programming language whose files are used in the evaluation.
+ * @property outputDir The output directory for the evaluation results.
+ * @property strategy The evaluation strategy used.
+ * @property actions The configuration for actions generation step.
+ * @property interpret The configuration for actions interpretation step.
+ * @property reorder The configuration for element reordering step.
+ * @property reports The configuration for report generation step.
+ */
 data class Config private constructor(
   val projectPath: String,
   val projectName: String,
@@ -35,10 +49,32 @@ data class Config private constructor(
     }
   }
 
+  /**
+   * Represents the configuration for generating actions.
+   *
+   * @property evaluationRoots The list of evaluation roots. Directories and files with relative and absolute paths are allowed.
+   * @property ignoreFileNames The set of file names to ignore. Files and directories with these names inside [evaluationRoots] will be skipped.
+   */
   data class ActionsGeneration internal constructor(
-    val evaluationRoots: List<String>
+    val evaluationRoots: List<String>,
+    val ignoreFileNames: Set<String>,
   )
 
+  /**
+   * Represents the configuration for the interpretation of actions.
+   *
+   * @property experimentGroup The ID of A/B experiment group.
+   * @property sessionsLimit The limit of sessions in the evaluation.
+   * @property filesLimit The limit of files in the evaluation.
+   * @property sessionProbability The probability of a session being evaluated.
+   * @property sessionSeed The seed for the random session sampling.
+   * @property order The order of session interpretation.
+   * @property saveLogs Whether to save logs.
+   * @property saveFeatures Whether to save ML features for rendering them in reports.
+   * @property saveContent Whether to save the content of files.
+   * @property logLocationAndItemText Whether to log location and item text in detailed ranking logs.
+   * @property trainTestSplit The train test split for detailed ranking logs.
+   */
   data class ActionsInterpretation internal constructor(
     val experimentGroup: Int?,
     val sessionsLimit: Int?,
@@ -52,12 +88,27 @@ data class Config private constructor(
     val logLocationAndItemText: Boolean,
     val trainTestSplit: Int)
 
+  /**
+   * Represents the configuration for reordering elements step.
+   *
+   * @property useReordering Whether to use element reordering.
+   * @property title The title of the reordering in reports.
+   * @property features The list of ML features to be used for reordering.
+   */
   data class ReorderElements internal constructor(
     val useReordering: Boolean,
     val title: String,
     val features: List<String>
   )
 
+  /**
+   * Represents the configuration for generating reports step.
+   *
+   * @property evaluationTitle The title of the evaluation.
+   * @property defaultMetrics The list of default metrics rendered in the report.
+   * @property sessionsFilters The list of session filters. These filters allow computing metrics and render reports on a subset of sessions.
+   * @property comparisonFilters The list of comparison filters. These filters allow subsetting sessions based on multiple evaluations.
+   */
   data class ReportGeneration internal constructor(
     val evaluationTitle: String,
     val defaultMetrics: List<String>?,
@@ -66,6 +117,7 @@ data class Config private constructor(
 
   class Builder internal constructor(private val projectPath: String, private val language: String) {
     var evaluationRoots = mutableListOf<String>()
+    var ignoreFileNames = mutableSetOf<String>()
     var projectName = projectPath.split('/').last()
     var outputDir: String = Paths.get(projectPath, "completion-evaluation").toAbsolutePath().toString()
     var strategy: EvaluationStrategy = EvaluationStrategy.defaultStrategy
@@ -93,6 +145,7 @@ data class Config private constructor(
       outputDir = config.outputDir
       strategy = config.strategy
       evaluationRoots.addAll(config.actions.evaluationRoots)
+      ignoreFileNames.addAll(config.actions.ignoreFileNames)
       saveLogs = config.interpret.saveLogs
       saveFeatures = config.interpret.saveFeatures
       saveContent = config.interpret.saveContent
@@ -131,7 +184,8 @@ data class Config private constructor(
       outputDir,
       strategy,
       ActionsGeneration(
-        evaluationRoots
+        evaluationRoots,
+        ignoreFileNames,
       ),
       ActionsInterpretation(
         experimentGroup,

plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt

@@ -1,4 +1,4 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.workspace
 
 import com.google.gson.*
@@ -66,6 +66,9 @@ object ConfigFactory {
   private fun deserializeActionsGeneration(map: Map<String, Any>?, language: String, builder: Config.Builder) {
     if (map == null) return
     builder.evaluationRoots = map.getAs("evaluationRoots")
+    if (map.containsKey("ignoreFileNames")) {
+      builder.ignoreFileNames = map.getAs<List<String>>("ignoreFileNames").toMutableSet()
+    }
   }
 
   private fun deserializeActionsInterpretation(map: Map<String, Any>?, builder: Config.Builder) {

plugins/evaluation-plugin/languages/src/com/intellij/cce/util/FilesHelper.kt

@@ -1,12 +1,9 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.util
 
 import com.intellij.cce.core.Language
-import com.intellij.openapi.fileTypes.FileTypeManager
-import com.intellij.openapi.fileTypes.LanguageFileType
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ContentIterator
-import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.openapi.vfs.VirtualFile
@@ -15,19 +12,20 @@ import java.io.FileNotFoundException
 import java.nio.file.Paths
 
 object FilesHelper {
-  fun getFilesOfLanguage(project: Project, evaluationRoots: List<String>, language: String): List<VirtualFile> {
-    return getFiles(project, evaluationRoots.map { getFile(project, it) })[language]?.toList()
+  fun getFilesOfLanguage(project: Project,
+                         evaluationRoots: List<String>,
+                         ignoreFileNames: Set<String>,
+                         language: String): List<VirtualFile> {
+    return getFiles(evaluationRoots.map { getFile(project, it) }, ignoreFileNames)[language]?.toList()
            ?: throw IllegalArgumentException("No files for $language found")
   }
 
-  private fun getFiles(project: Project, evaluationRoots: List<VirtualFile>): Map<String, Set<VirtualFile>> {
+  private fun getFiles(evaluationRoots: List<VirtualFile>, ignoreFileNames: Set<String>): Map<String, Set<VirtualFile>> {
     val language2files = mutableMapOf<String, MutableSet<VirtualFile>>()
-    val index = ProjectRootManager.getInstance(project).fileIndex
     for (file in evaluationRoots) {
-      //val filter = if (file.extension == "java") GlobalSearchScope.projectScope(project) else GlobalSearchScope.everythingScope(project)
-      val filter = null
-      VfsUtilCore.iterateChildrenRecursively(file, filter, object : ContentIterator {
+      VfsUtilCore.iterateChildrenRecursively(file, null, object : ContentIterator {
         override fun processFile(fileOrDir: VirtualFile): Boolean {
+          if (fileOrDir.name in ignoreFileNames) return false
           val extension = fileOrDir.extension
           if (fileOrDir.isDirectory || extension == null) return true
 
@@ -35,24 +33,11 @@ object FilesHelper {
           language2files.computeIfAbsent(language.displayName) { mutableSetOf() }.add(fileOrDir)
           return true
         }
-
-        private fun shouldEvaluateOnFile(language: Language, fileOrDir: VirtualFile): Boolean {
-          if (language == Language.JAVA || language == Language.KOTLIN) {
-            return index.isInSourceContent(fileOrDir)
-          }
-
-          return true
-        }
       })
     }
     return language2files
   }
 
-  fun getLanguageByExtension(ext: String): com.intellij.lang.Language? {
-    val fileType = FileTypeManager.getInstance().getFileTypeByExtension(ext) as? LanguageFileType ?: return null
-    return fileType.language
-  }
-
   fun getRelativeToProjectPath(project: Project, path: String): String {
     val projectPath = project.basePath
     return if (projectPath == null) path else Paths.get(projectPath).relativize(Paths.get(path)).toString()

plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt

@@ -75,7 +75,7 @@ internal class ContextCollectionEvaluationCommand: CompletionEvaluationStarter.E
           ) {
             override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace {
               val files = runReadAction {
-                FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, language)
+                FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, config.actions.ignoreFileNames, language)
               }.sortedBy { it.name }
               val strategy = config.strategy as CompletionContextCollectionStrategy
               val sampled = files.shuffled(Random(strategy.samplingSeed)).take(strategy.samplesCount).sortedBy { it.name }

plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsGenerationStep.kt

@@ -1,4 +1,4 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.cce.evaluation.step
 
 import com.intellij.cce.actions.ActionsGenerator
@@ -38,7 +38,7 @@ open class ActionsGenerationStep(
 
   override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace {
     val filesForEvaluation = ReadAction.compute<List<VirtualFile>, Throwable> {
-      FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, language)
+      FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, config.actions.ignoreFileNames, language)
     }
     generateActions(workspace, language, filesForEvaluation, evaluationRootInfo, config.interpret.filesLimit, progress)
     return workspace