From 43d9722b6e76063c084caf3af23207cff3a91505 Mon Sep 17 00:00:00 2001 From: Alexey Kalina Date: Tue, 13 Feb 2024 11:01:16 +0100 Subject: [PATCH] [evaluation-plugin] ML-2550 Option to exclude a folder from eval GitOrigin-RevId: a303ab8b4ee385dc08fa9e64e689fd1086da33e8 --- plugins/evaluation-plugin/README.md | 1 + .../src/com/intellij/cce/workspace/Config.kt | 60 ++++++++++++++++++- .../intellij/cce/workspace/ConfigFactory.kt | 5 +- .../src/com/intellij/cce/util/FilesHelper.kt | 33 +++------- .../ContextCollectionEvaluationCommand.kt | 2 +- .../evaluation/step/ActionsGenerationStep.kt | 4 +- 6 files changed, 74 insertions(+), 31 deletions(-) diff --git a/plugins/evaluation-plugin/README.md b/plugins/evaluation-plugin/README.md index 8df898632ef7..7bb3eabb4531 100644 --- a/plugins/evaluation-plugin/README.md +++ b/plugins/evaluation-plugin/README.md @@ -107,6 +107,7 @@ Here is an example of such file with description for possible options but the st }, "actions": { // part of config about actions generation step "evaluationRoots": [], // list of string with paths to files/directories for evaluation + "ignoreFileNames": [] // list of file/directory names to be ignored inside evaluationRoots }, "interpret": { // part of config about actions interpretation step "sessionProbability": 1.0, // probability that session won't be skipped diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt index 624a3347aa2b..ad7cc2aa3a79 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.cce.workspace import com.intellij.cce.evaluable.EvaluationStrategy @@ -10,6 +10,20 @@ import com.intellij.cce.workspace.filter.SessionsFilter import java.nio.file.Paths import kotlin.io.path.absolute + +/** + * Represents a configuration of the evaluation process. + * + * @property projectPath The path to the project that will be used for the evaluation. + * @property projectName The name of the project. It may differ from the root directory name. + * @property language The programming language whose files are used in the evaluation. + * @property outputDir The output directory for the evaluation results. + * @property strategy The evaluation strategy used. + * @property actions The configuration for actions generation step. + * @property interpret The configuration for actions interpretation step. + * @property reorder The configuration for element reordering step. + * @property reports The configuration for report generation step. + */ data class Config private constructor( val projectPath: String, val projectName: String, @@ -35,10 +49,32 @@ data class Config private constructor( } } + /** + * Represents the configuration for generating actions. + * + * @property evaluationRoots The list of evaluation roots. Directories and files with relative and absolute paths are allowed. + * @property ignoreFileNames The set of file names to ignore. Files and directories with these names inside [evaluationRoots] will be skipped. + */ data class ActionsGeneration internal constructor( - val evaluationRoots: List + val evaluationRoots: List, + val ignoreFileNames: Set, ) + /** + * Represents the configuration for the interpretation of actions. + * + * @property experimentGroup The ID of A/B experiment group. + * @property sessionsLimit The limit of sessions in the evaluation. + * @property filesLimit The limit of files in the evaluation. + * @property sessionProbability The probability of a session being evaluated. + * @property sessionSeed The seed for the random session sampling. + * @property order The order of session interpretation. + * @property saveLogs Whether to save logs. + * @property saveFeatures Whether to save ML features for rendering them in reports. + * @property saveContent Whether to save the content of files. + * @property logLocationAndItemText Whether to log location and item text in detailed ranking logs. + * @property trainTestSplit The train test split for detailed ranking logs. + */ data class ActionsInterpretation internal constructor( val experimentGroup: Int?, val sessionsLimit: Int?, @@ -52,12 +88,27 @@ data class Config private constructor( val logLocationAndItemText: Boolean, val trainTestSplit: Int) + /** + * Represents the configuration for reordering elements step. + * + * @property useReordering Whether to use element reordering. + * @property title The title of the reordering in reports. + * @property features The list of ML features to be used for reordering. + */ data class ReorderElements internal constructor( val useReordering: Boolean, val title: String, val features: List ) + /** + * Represents the configuration for generating reports step. + * + * @property evaluationTitle The title of the evaluation. + * @property defaultMetrics The list of default metrics rendered in the report. + * @property sessionsFilters The list of session filters. These filters allow computing metrics and render reports on a subset of sessions. + * @property comparisonFilters The list of comparison filters. These filters allow subsetting sessions based on multiple evaluations. + */ data class ReportGeneration internal constructor( val evaluationTitle: String, val defaultMetrics: List?, @@ -66,6 +117,7 @@ data class Config private constructor( class Builder internal constructor(private val projectPath: String, private val language: String) { var evaluationRoots = mutableListOf() + var ignoreFileNames = mutableSetOf() var projectName = projectPath.split('/').last() var outputDir: String = Paths.get(projectPath, "completion-evaluation").toAbsolutePath().toString() var strategy: EvaluationStrategy = EvaluationStrategy.defaultStrategy @@ -93,6 +145,7 @@ data class Config private constructor( outputDir = config.outputDir strategy = config.strategy evaluationRoots.addAll(config.actions.evaluationRoots) + ignoreFileNames.addAll(config.actions.ignoreFileNames) saveLogs = config.interpret.saveLogs saveFeatures = config.interpret.saveFeatures saveContent = config.interpret.saveContent @@ -131,7 +184,8 @@ data class Config private constructor( outputDir, strategy, ActionsGeneration( - evaluationRoots + evaluationRoots, + ignoreFileNames, ), ActionsInterpretation( experimentGroup, diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt index 53630e6c3482..b668ac8e0c61 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.cce.workspace import com.google.gson.* @@ -66,6 +66,9 @@ object ConfigFactory { private fun deserializeActionsGeneration(map: Map?, language: String, builder: Config.Builder) { if (map == null) return builder.evaluationRoots = map.getAs("evaluationRoots") + if (map.containsKey("ignoreFileNames")) { + builder.ignoreFileNames = map.getAs>("ignoreFileNames").toMutableSet() + } } private fun deserializeActionsInterpretation(map: Map?, builder: Config.Builder) { diff --git a/plugins/evaluation-plugin/languages/src/com/intellij/cce/util/FilesHelper.kt b/plugins/evaluation-plugin/languages/src/com/intellij/cce/util/FilesHelper.kt index 1eff3467e4a8..04430c96c4cf 100644 --- a/plugins/evaluation-plugin/languages/src/com/intellij/cce/util/FilesHelper.kt +++ b/plugins/evaluation-plugin/languages/src/com/intellij/cce/util/FilesHelper.kt @@ -1,12 +1,9 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.cce.util import com.intellij.cce.core.Language -import com.intellij.openapi.fileTypes.FileTypeManager -import com.intellij.openapi.fileTypes.LanguageFileType import com.intellij.openapi.project.Project import com.intellij.openapi.roots.ContentIterator -import com.intellij.openapi.roots.ProjectRootManager import com.intellij.openapi.vfs.VfsUtil import com.intellij.openapi.vfs.VfsUtilCore import com.intellij.openapi.vfs.VirtualFile @@ -15,19 +12,20 @@ import java.io.FileNotFoundException import java.nio.file.Paths object FilesHelper { - fun getFilesOfLanguage(project: Project, evaluationRoots: List, language: String): List { - return getFiles(project, evaluationRoots.map { getFile(project, it) })[language]?.toList() + fun getFilesOfLanguage(project: Project, + evaluationRoots: List, + ignoreFileNames: Set, + language: String): List { + return getFiles(evaluationRoots.map { getFile(project, it) }, ignoreFileNames)[language]?.toList() ?: throw IllegalArgumentException("No files for $language found") } - private fun getFiles(project: Project, evaluationRoots: List): Map> { + private fun getFiles(evaluationRoots: List, ignoreFileNames: Set): Map> { val language2files = mutableMapOf>() - val index = ProjectRootManager.getInstance(project).fileIndex for (file in evaluationRoots) { - //val filter = if (file.extension == "java") GlobalSearchScope.projectScope(project) else GlobalSearchScope.everythingScope(project) - val filter = null - VfsUtilCore.iterateChildrenRecursively(file, filter, object : ContentIterator { + VfsUtilCore.iterateChildrenRecursively(file, null, object : ContentIterator { override fun processFile(fileOrDir: VirtualFile): Boolean { + if (fileOrDir.name in ignoreFileNames) return false val extension = fileOrDir.extension if (fileOrDir.isDirectory || extension == null) return true @@ -35,24 +33,11 @@ object FilesHelper { language2files.computeIfAbsent(language.displayName) { mutableSetOf() }.add(fileOrDir) return true } - - private fun shouldEvaluateOnFile(language: Language, fileOrDir: VirtualFile): Boolean { - if (language == Language.JAVA || language == Language.KOTLIN) { - return index.isInSourceContent(fileOrDir) - } - - return true - } }) } return language2files } - fun getLanguageByExtension(ext: String): com.intellij.lang.Language? { - val fileType = FileTypeManager.getInstance().getFileTypeByExtension(ext) as? LanguageFileType ?: return null - return fileType.language - } - fun getRelativeToProjectPath(project: Project, path: String): String { val projectPath = project.basePath return if (projectPath == null) path else Paths.get(projectPath).relativize(Paths.get(path)).toString() diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt index f97ebc0bca12..9dafd6d91f7b 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt @@ -75,7 +75,7 @@ internal class ContextCollectionEvaluationCommand: CompletionEvaluationStarter.E ) { override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace { val files = runReadAction { - FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, language) + FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, config.actions.ignoreFileNames, language) }.sortedBy { it.name } val strategy = config.strategy as CompletionContextCollectionStrategy val sampled = files.shuffled(Random(strategy.samplingSeed)).take(strategy.samplesCount).sortedBy { it.name } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsGenerationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsGenerationStep.kt index 5ba35c2145c1..d4b1b5ea9241 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsGenerationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsGenerationStep.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.cce.evaluation.step import com.intellij.cce.actions.ActionsGenerator @@ -38,7 +38,7 @@ open class ActionsGenerationStep( override fun runInBackground(workspace: EvaluationWorkspace, progress: Progress): EvaluationWorkspace { val filesForEvaluation = ReadAction.compute, Throwable> { - FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, language) + FilesHelper.getFilesOfLanguage(project, config.actions.evaluationRoots, config.actions.ignoreFileNames, language) } generateActions(workspace, language, filesForEvaluation, evaluationRootInfo, config.interpret.filesLimit, progress) return workspace