diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt index 0b17a148eecc..215eb99ac1c6 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluable/AiaConstants.kt @@ -40,4 +40,5 @@ const val AIA_CORRECT_ATTACHMENT: String = "aia_correct_attachment" const val AIA_RETRIEVED_ATTACHMENTS: String = "aia_retrieved_attachments" const val AIA_IS_CORRECT_ATTACHMENT_AMONG_RETRIEVED: String = "aia_is_correct_attachment_among_retrieved" const val AIA_EVAL_ARTIFACT: String = "aia_eval_artifact" -const val AIA_EVAL_ARTIFACT_NAME: String = "aia_eval_artifact_name" \ No newline at end of file +const val AIA_EVAL_ARTIFACT_NAME: String = "aia_eval_artifact_name" +const val AIA_EVAL_ARTIFACT_FILES: String = "aia_eval_artifact_files" diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt index cf130442deca..921df502983f 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/DataPlacement.kt @@ -132,6 +132,22 @@ sealed interface DataPlacement { } } + data class ArtifactFiles(val propertyKey: String) : DataPlacement, ArtifactFile> { + override val serialName: String = "artifact_files" + + override fun dump(lookup: Lookup, t: List): Lookup { + return lookup.copy( + additionalInfo = lookup.additionalInfo + Pair(propertyKey, gson.toJsonTree(t)) + ) + } + + override fun restore(props: DataProps): List { + val files = props.lookup.additionalInfo[propertyKey] ?: return emptyList() + val filesJson = files as? JsonElement ?: gson.toJsonTree(files) + return gson.fromJson(filesJson, Array::class.java).toList() + } + } + data class AdditionalCodeCommentRanges(val propertyKey: String) : DataPlacement, List> { override val serialName: String = "code_comment_range" @@ -185,6 +201,7 @@ sealed interface DataPlacement { "latency" -> Latency "current_file_update" -> CurrentFileUpdate "file_updates" -> context?.deserialize(json, FileUpdates::class.java) + "artifact_files" -> context?.deserialize(json, ArtifactFiles::class.java) "colored_insights_placement" -> context?.deserialize(json, ColoredInsightsPlacement::class.java) else -> throw IllegalArgumentException("Unknown type: $type") } @@ -205,4 +222,4 @@ data class DataProps( init { check(session.lookups.contains(lookup)) } -} \ No newline at end of file +} diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt index 9345c4a6b20f..1fd15092d0c1 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/evaluation/data/EvalDataConstants.kt @@ -11,6 +11,7 @@ import com.intellij.cce.evaluable.AIA_CORRECT_ATTACHMENT import com.intellij.cce.evaluable.AIA_DESCRIPTION import com.intellij.cce.evaluable.AIA_ERASED_APIS import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT +import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_FILES import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_NAME import com.intellij.cce.evaluable.AIA_EXACT_MATCH import com.intellij.cce.evaluable.AIA_EXPECTED_FUNCTION_CALLS @@ -288,8 +289,19 @@ object Execution { ignoreMissingData = true, ) ) + + val ARTIFACTS_FILES: EvalDataDescription, ArtifactFile> = EvalDataDescription( + name = "Artifact files", + description = "Files that should be copied to report artifacts. Source is the path to copy, target is the relative report path", + placement = DataPlacement.ArtifactFiles(AIA_EVAL_ARTIFACT_FILES), + ) } +data class ArtifactFile( + val target: String, + val source: String, +) + object Analysis { val HAS_SYNTAX_ERRORS: TrivialEvalData = EvalDataDescription( name = "Has syntax errors", diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/ArtifactsToReport.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/ArtifactsToReport.kt index c3d6cb60c762..c53a16f19f5b 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/ArtifactsToReport.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/ArtifactsToReport.kt @@ -2,9 +2,13 @@ package com.intellij.cce.report import com.intellij.cce.core.Lookup +import com.intellij.cce.core.Session import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT import com.intellij.cce.evaluable.AIA_EVAL_ARTIFACT_NAME import com.intellij.cce.evaluable.AIA_NAME +import com.intellij.cce.evaluation.data.ArtifactFile +import com.intellij.cce.evaluation.data.DataProps +import com.intellij.cce.evaluation.data.Execution import com.intellij.cce.metric.MetricInfo import com.intellij.cce.workspace.info.FileErrorInfo import com.intellij.cce.workspace.info.FileEvaluationInfo @@ -12,6 +16,7 @@ import com.intellij.openapi.diagnostic.Logger import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.StandardCopyOption class ArtifactsToReport( outputDir: String, @@ -21,7 +26,7 @@ class ArtifactsToReport( override val type: String = "artifacts" - private val dir: Path = Paths.get(outputDir, comparisonFilterName, type, filterName).also { Files.createDirectories(it) } + private val dir: Path = Paths.get(outputDir, comparisonFilterName, type, filterName).normalize().also { Files.createDirectories(it) } override fun generateFileReport(sessions: List) { if (sessions.isEmpty()) return @@ -30,6 +35,7 @@ class ArtifactsToReport( for (session in fileEvaluation.sessionsInfo.sessions) { for (lookup in session.lookups) { saveArtifact(lookup) + saveArtifactFiles(session, lookup) } } } @@ -54,6 +60,39 @@ class ArtifactsToReport( Files.writeString(artifactPath, artifact) } + private fun saveArtifactFiles(session: Session, lookup: Lookup) { + val props = DataProps(null, null, session, lookup) + for (artifactFile in Execution.ARTIFACTS_FILES.placement.restore(props)) { + copyArtifactFile(artifactFile) + } + } + + private fun copyArtifactFile(artifactFile: ArtifactFile) { + val targetPath = resolveTarget(artifactFile.target) ?: return + targetPath.parent?.let { Files.createDirectories(it) } + Files.copy(Paths.get(artifactFile.source), targetPath, StandardCopyOption.REPLACE_EXISTING) + } + + private fun resolveTarget(target: String): Path? { + if (target.isBlank()) { + LOG.warn("Failed to save artifact file: target path is blank") + return null + } + + val relativeTarget = Paths.get(target) + if (relativeTarget.isAbsolute) { + LOG.warn("Failed to save artifact file: target path should be relative: $target") + return null + } + + val targetPath = dir.resolve(relativeTarget).normalize() + if (!targetPath.startsWith(dir)) { + LOG.warn("Failed to save artifact file: target path escapes report directory: $target") + return null + } + return targetPath + } + override fun generateErrorReports(errors: List): Unit = Unit override fun generateGlobalReport(globalMetrics: List): Path { diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt index f6804dac48ea..87967965c9a4 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/report/CardReportGenerator.kt @@ -430,6 +430,10 @@ private data class PropertyValue( """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].originalText""", """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].updatedText""" ) + is DataPlacement.ArtifactFiles -> listOf( + """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].target""", + """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].source""" + ) is DataPlacement.AdditionalCodeCommentRanges -> listOf( """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].start""", """sessions["${sessionId}"]["_lookups"][${lookupIndex}]["additionalInfo"]["${placement.propertyKey}"][${placementIndex}].end""", @@ -487,4 +491,4 @@ data class CardLayout( builder.create() } } -} \ No newline at end of file +} diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/report/ArtifactsToReportTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/report/ArtifactsToReportTest.kt new file mode 100644 index 000000000000..6aa82ccafff7 --- /dev/null +++ b/plugins/evaluation-plugin/test/com/intellij/cce/report/ArtifactsToReportTest.kt @@ -0,0 +1,69 @@ +// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.cce.report + +import com.intellij.cce.core.Lookup +import com.intellij.cce.core.Session +import com.intellij.cce.core.TokenProperties +import com.intellij.cce.evaluation.data.ArtifactFile +import com.intellij.cce.evaluation.data.Execution +import com.intellij.cce.workspace.info.FileEvaluationInfo +import com.intellij.cce.workspace.info.FileSessionsInfo +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class ArtifactsToReportTest { + @Test + fun `copies artifact files to report directory`(@TempDir tempDir: Path) { + val source = tempDir.resolve("source.txt") + Files.writeString(source, "artifact file content") + + val lookup = Execution.ARTIFACTS_FILES.placement.dump( + lookup(), + listOf(ArtifactFile(target = "nested/result.txt", source = source.toString())) + ) + + ArtifactsToReport(tempDir.resolve("report").toString(), "filter", "comparison") + .generateFileReport(listOf(fileEvaluationInfo(lookup))) + + val copied = tempDir.resolve("report/comparison/artifacts/filter/nested/result.txt") + assertEquals("artifact file content", Files.readString(copied)) + } + + @Test + fun `does not copy artifact files outside report directory`(@TempDir tempDir: Path) { + val source = tempDir.resolve("source.txt") + Files.writeString(source, "artifact file content") + + val lookup = Execution.ARTIFACTS_FILES.placement.dump( + lookup(), + listOf(ArtifactFile(target = "../outside.txt", source = source.toString())) + ) + + ArtifactsToReport(tempDir.resolve("report").toString(), "filter", "comparison") + .generateFileReport(listOf(fileEvaluationInfo(lookup))) + + assertFalse(Files.exists(tempDir.resolve("report/comparison/artifacts/outside.txt"))) + } + + private fun fileEvaluationInfo(lookup: Lookup): FileEvaluationInfo { + val session = Session(0, "", 0, TokenProperties.UNKNOWN) + session.addLookup(lookup) + return FileEvaluationInfo( + sessionsInfo = FileSessionsInfo("project", "file.kt", "", listOf(session)), + metrics = emptyList(), + evaluationType = "evaluation" + ) + } + + private fun lookup(): Lookup = Lookup( + prefix = "", + offset = 0, + suggestions = emptyList(), + latency = 0, + isNew = true + ) +}