From a6c3fa7948749835f29a75972bcf3a7ea6645c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20=C3=96zerbay?= Date: Fri, 7 Feb 2025 08:24:30 +0000 Subject: [PATCH] [LME-311] Support multi file validation checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduced functionality to compute validations for related files by comparing initial and result states. Enhanced `CodeGenerationInvoker` to parse related files and manage their validations, ensuring more accurate evaluation across multiple files. This improves validation coverage for auxiliary file changes during code generation. Merge-request: IJ-MR-153911 Merged-by: Berkay Özerbay GitOrigin-RevId: 60f3d045da7e28eedbc429eb35d15082ef400ebf --- .../actions/simplified/DatasetTransformer.kt | 18 ++++-------------- .../simplified/SimplifiedDatasetSerializer.kt | 10 ++++++++++ .../com/intellij/cce/evaluable/AiaConstants.kt | 1 + .../com/intellij/cce/evaluable/Constants.kt | 1 + .../simplified/DatasetTransformerTest.kt | 8 ++------ 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/DatasetTransformer.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/DatasetTransformer.kt index f867fc446dc2..86ad44984c24 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/DatasetTransformer.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/DatasetTransformer.kt @@ -5,10 +5,8 @@ import com.intellij.cce.actions.simplified.SimplifiedDatasetSerializer.parseLine import com.intellij.cce.core.SimpleTokenProperties import com.intellij.cce.core.SymbolLocation import com.intellij.cce.core.TypeProperty -import com.intellij.cce.evaluable.FILE_CHANGED_LINES_PREFIX -import com.intellij.cce.evaluable.FILE_PATTERN_PROPERTY_PREFIX -import com.intellij.cce.evaluable.FILE_UNCHANGED_LINES_PREFIX import com.intellij.cce.evaluable.PROMPT_PROPERTY +import com.intellij.cce.evaluable.FILE_VALIDATIONS import java.nio.file.Files import java.nio.file.Paths import java.util.* @@ -61,18 +59,10 @@ class DatasetTransformer(private val offsetProvider: OffsetProvider) { } private fun generateActions(sessionId: UUID, interaction: SimplifiedDatasetSerializer.InteractionData): List { - val fileValidation = interaction.fileValidations.firstOrNull() + val fileValidationsJson = SimplifiedDatasetSerializer.serializeFileValidations(interaction.fileValidations) val nodeProperties = SimpleTokenProperties.create(TypeProperty.FILE, SymbolLocation.PROJECT) { - put(PROMPT_PROPERTY,interaction.userPrompt) - fileValidation?.patterns.orEmpty().forEachIndexed { i, pattern -> - put("${FILE_PATTERN_PROPERTY_PREFIX}_${i+1}", pattern) - } - fileValidation?.changedLines.orEmpty().forEachIndexed { i, pattern -> - put("${FILE_CHANGED_LINES_PREFIX}_${i+1}", pattern) - } - fileValidation?.unchangedLines.orEmpty().forEachIndexed { i, pattern -> - put("${FILE_UNCHANGED_LINES_PREFIX}_${i+1}", pattern) - } + put(PROMPT_PROPERTY, interaction.userPrompt) + put(FILE_VALIDATIONS, fileValidationsJson) } val position = interaction.position val defaultLine = position.caretLine ?: parseLineRange(position.selectionLines)?.first ?: 0 diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/SimplifiedDatasetSerializer.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/SimplifiedDatasetSerializer.kt index 496180102f26..a307fc319fa5 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/SimplifiedDatasetSerializer.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/actions/simplified/SimplifiedDatasetSerializer.kt @@ -1,10 +1,20 @@ package com.intellij.cce.actions.simplified import com.google.gson.GsonBuilder +import com.google.gson.reflect.TypeToken object SimplifiedDatasetSerializer { private val gson = GsonBuilder().create() + fun parseFileValidations(jsonData: String): List { + val listType = object : TypeToken>() {}.type + return gson.fromJson(jsonData, listType) + } + + fun serializeFileValidations(fileValidations: List): String { + return gson.toJson(fileValidations) + } + /** * Parses a string representing a range of integers in the format "start-end" and returns it as an IntRange object. */ diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt index 2d24f43495af..f8731c93e4c6 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt @@ -5,6 +5,7 @@ const val AIA_USER_PROMPT = "aia_user_prompt" const val AIA_RESPONSE = "aia_response" const val AIA_ORIGINAL_CONTENT = "aia_original_content" const val AIA_FAILED_FILE_VALIDATIONS = "aia_failed_file_validations" +const val AIA_FAILED_RELATED_FILE_VALIDATIONS = "aia_failed_related_file_validations" const val AIA_HAS_SYNTAX_ERRORS = "has_syntax_errors" const val AIA_HAS_HIGHLIGHT_ERRORS = "has_highlight_errors" const val AIA_PREDICTED_API_CALLS = "predicted_api_calls" diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt index 2327d48a84af..47ac09ad52d9 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/Constants.kt @@ -1,6 +1,7 @@ package com.intellij.cce.evaluable const val PROMPT_PROPERTY = "prompt" +const val FILE_VALIDATIONS = "file_validations" const val SNIPPET_PROMPT_PROPERTY = "snippet_prompt" const val PROMPT_PRESENTATION_PROPERTY = "prompt_presentation" const val INTERNAL_API_CALLS_PROPERTY = "internal_api_calls" diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/actions/simplified/DatasetTransformerTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/actions/simplified/DatasetTransformerTest.kt index 0e4b62849ee0..1a2762600816 100644 --- a/plugins/evaluation-plugin/test/com/intellij/cce/actions/simplified/DatasetTransformerTest.kt +++ b/plugins/evaluation-plugin/test/com/intellij/cce/actions/simplified/DatasetTransformerTest.kt @@ -73,9 +73,7 @@ class DatasetTransformerTest { "features": [], "additional": { "prompt": "add method to check email string", - "file_property_regex_1": "static.*boolean.*mail.*String", - "file_changed_lines_1": "5-6", - "file_unchanged_lines_1": "1-3" + "file_validations": "[{\"path\":\"src/main/java/com/jetbrains/eval/Utils.java\",\"patterns\":[\"static.*boolean.*mail.*String\"],\"changedLines\":[\"5-6\"],\"unchangedLines\":[\"1-3\"]}]" } }, "type": "CALL_FEATURE" @@ -101,9 +99,7 @@ class DatasetTransformerTest { "features": [], "additional": { "prompt": "add method to check email string", - "file_property_regex_1": "static.*boolean.*mail.*String", - "file_changed_lines_1": "5-6", - "file_unchanged_lines_1": "1-3" + "file_validations": "[{\"path\":\"src/main/java/com/jetbrains/eval/Utils.java\",\"patterns\":[\"static.*boolean.*mail.*String\"],\"changedLines\":[\"5-6\"],\"unchangedLines\":[\"1-3\"]}]" } }, "type": "CALL_FEATURE"